From ea09aca5ed61d1b3c7b9b340946662d17bf4fa37 Mon Sep 17 00:00:00 2001 From: gspark Date: Fri, 23 Oct 2026 23:54:41 +0900 Subject: [PATCH] test: validate four-model live GPU PD policies and lifecycle (#240) --- opt_prime/examples/pd/http_matrix/README.md | 14 + opt_prime/examples/pd/http_matrix/STATUS.md | 4 + .../pd/http_matrix/evidence240/events.jsonl | 3 + ...gpu240-cli-qwen06b-http-module-failed.json | 762 +++++++++++++++++ .../evidence240/gpu240-cli-qwen06b.json | 777 ++++++++++++++++++ .../evidence240/gpu240-gemma-native-fp32.json | 770 +++++++++++++++++ .../evidence240/gpu240-gemma-native.json | 770 +++++++++++++++++ .../evidence240/gpu240-gemma-pd-cacheoff.json | 777 ++++++++++++++++++ .../evidence240/gpu240-gemma-pd-chunk256.json | 777 ++++++++++++++++++ .../evidence240/gpu240-gemma-pd-fp32.json | 777 ++++++++++++++++++ .../evidence240/gpu240-gemma-pd.json | 776 +++++++++++++++++ .../evidence240/gpu240-koni-native-fp32.json | 770 +++++++++++++++++ .../evidence240/gpu240-koni-native.json | 770 +++++++++++++++++ .../evidence240/gpu240-koni-pd-cacheoff.json | 777 ++++++++++++++++++ .../evidence240/gpu240-koni-pd-fp32.json | 777 ++++++++++++++++++ .../evidence240/gpu240-koni-pd.json | 776 +++++++++++++++++ .../gpu240-qwen06b-native-fp32.json | 770 +++++++++++++++++ .../evidence240/gpu240-qwen06b-native.json | 770 +++++++++++++++++ .../gpu240-qwen06b-pd-cacheoff.json | 777 ++++++++++++++++++ .../gpu240-qwen06b-pd-chunk256.json | 777 ++++++++++++++++++ .../evidence240/gpu240-qwen06b-pd-fp32.json | 777 ++++++++++++++++++ .../evidence240/gpu240-qwen06b-pd.json | 776 +++++++++++++++++ .../evidence240/gpu240-qwen4b-native.json | 770 +++++++++++++++++ .../evidence240/gpu240-qwen4b-pd.json | 776 +++++++++++++++++ .../evidence240/gpu240-summary.json | 82 ++ .../experiments/2026-10-23-gpu-live-pd240.md | 51 ++ opt_prime/examples/pd/http_matrix/validate.py | 39 +- 27 files changed, 16437 insertions(+), 5 deletions(-) create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/events.jsonl create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-cli-qwen06b-http-module-failed.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-cli-qwen06b.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-native-fp32.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-native.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-pd-cacheoff.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-pd-chunk256.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-pd-fp32.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-pd.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-koni-native-fp32.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-koni-native.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-koni-pd-cacheoff.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-koni-pd-fp32.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-koni-pd.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-native-fp32.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-native.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-pd-cacheoff.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-pd-chunk256.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-pd-fp32.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-pd.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen4b-native.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen4b-pd.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence240/gpu240-summary.json create mode 100644 opt_prime/examples/pd/http_matrix/experiments/2026-10-23-gpu-live-pd240.md diff --git a/opt_prime/examples/pd/http_matrix/README.md b/opt_prime/examples/pd/http_matrix/README.md index e2b28516..038d7625 100644 --- a/opt_prime/examples/pd/http_matrix/README.md +++ b/opt_prime/examples/pd/http_matrix/README.md @@ -31,3 +31,17 @@ python examples/pd/serve_batched.py --model "$MODEL" --device cuda:1 --slots 4 \ ``` The new flags affect the in-process Torch producer only. Nonpositive producer chunks are rejected. Disabling reuse or using smaller chunks can increase prefill work/overhead; these options do not promise a speedup or universal bitwise equality. CPU factory and four-model GPU controls are recorded under issue #241. + +### Live GPU PD matrix + +The validator accepts `--device`, `--pd-prefill-device` and the producer controls above, records actual producer policy and asserts remote prefill/transfer counters. It also accepts `--http-module` for a trusted local HTTP implementation candidate; the candidate is loaded before its consumers so exception classes retain their identity. + +```bash +git show 9e442cd:opt_prime/opt_prime/accel/serve/llmd.py > /tmp/pd-http-safe.py +PYTHONPATH=opt_prime python opt_prime/examples/pd/http_matrix/validate.py \ + --http-module /tmp/pd-http-safe.py --model "$MODEL" --output "$OUTPUT" \ + --device cuda:1 --pd-prefill-device cuda:0 \ + --pd-prefill-no-prefix-cache --pd-prefill-chunk 256 +``` + +Run from the repository root. Omit the producer arguments for native control. `passed` means lifecycle/count assertions passed; concurrent/serial token equality is reported separately, including failed numerical comparisons. See gpu-live-pd240 for the complete default-policy, FP32 and matching-policy results. diff --git a/opt_prime/examples/pd/http_matrix/STATUS.md b/opt_prime/examples/pd/http_matrix/STATUS.md index 4d6e5fe7..cfc3154f 100644 --- a/opt_prime/examples/pd/http_matrix/STATUS.md +++ b/opt_prime/examples/pd/http_matrix/STATUS.md @@ -25,3 +25,7 @@ Structured SDK logs and cancelled/malformed replies previously killed the receiv ## Explicit Torch producer policy (#241) New opt-in controls disable producer prefix reuse and select producer chunk size; defaults remain cache-on/chunk2048. Public cache-off/chunk256 configuration passes all four real GPU PD HTTP paths:48/48 concurrent equality,12/12 native-PD serial equality at128/2048/8192tokens,84remote prefills and0producer hits. Lifecycle checks pass.20focused and464full tests pass (15skipped,135subtests). This is tested numerical reproducibility, not a speedup or arbitrary-input equality guarantee; original default-policy discrepancies remain in#240. + +## Live GPU PD matrix (#240) + +All four native/PD BF16 lifecycle configurations pass across128/2048/8192tokens. Default PD has35/48concurrent equality versus native48/48, and11/12native-PD serial equality. FP32 restores all divergent models; BF16 explicit cache-off/chunk256 controls#241 restore four-model48/48concurrent and12/12serial equality. Original divergences and validator module-identity failure are retained in evidence240. The self-contained HTTP-candidate loader was fixed and Qwen0.6B rerun passes. No matched speedup or compute/copy overlap claim. diff --git a/opt_prime/examples/pd/http_matrix/evidence240/events.jsonl b/opt_prime/examples/pd/http_matrix/evidence240/events.jsonl new file mode 100644 index 00000000..ac6e9306 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/events.jsonl @@ -0,0 +1,3 @@ +{"id": "a4854a4bbae8442ea6c1a5b3ce82e7ba", "ts": "2026-10-23T23:33:18+09:00", "task": "GPU live PD matrix (aicomp #240)", "result": "Qwen3-0.6B BF16 GPU0->GPU1 HTTP lifecycle passes;21remote prefills/44544transferred tokens. Concurrent equality8/12, retained as numerical discrepancy. Remaining native/PD queue49704 active.", "next": "Complete four-model BF16 native/PD matrix, then FP32 controls; preserve all numerical failures.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"dtype": "bfloat16", "decode_slots": "4"}, "artifacts": [{"path": "analysis/serving-matrix-201/gpu240-qwen06b-pd.json", "size": 24146, "sha256": "84de5338ae7a227f"}], "experiments": ["experiments/2026-10-23-gpu-live-pd240.md"], "labels": ["experiment"], "review": false, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"} +{"id": "bb809b6ecfa54c1c913cdbbaff56c857", "ts": "2026-10-23T23:36:05+09:00", "task": "GPU live PD matrix (aicomp #240)", "result": "Four-model BF16 native+PD8/8 lifecycle runs pass,84remote prefills. Native concurrency48/48 exact; PD35/48 exact(Q06=8,Q4=12,KONI=7,Gemma=8 of12). Native/PD serial11/12; Gemma2K differs.", "next": "Run FP32 and producer-cache-off controls before attributing numerical differences; keep full goal active.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"dtype": "bfloat16", "decode_slots": "4"}, "artifacts": [{"path": "analysis/serving-matrix-201/gpu240-summary.json", "size": 1555, "sha256": "46799a4baa76b48e"}], "experiments": ["experiments/2026-10-23-gpu-live-pd240.md"], "labels": ["experiment"], "review": false, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"} +{"id": "75a2497a5045452ca4660b574cf6211f", "ts": "2026-10-23T23:44:56+09:00", "task": "GPU live PD matrix (aicomp #240)", "result": "FP32 controls restore Q06/KONI/Gemma36/36 concurrent and9/9 native-PD serial equality. BF16 cache-off gives36/36 concurrent but7/9 serial; matching chunk256 restores Q06/Gemma6/6 serial. Public-policy4model queue20549 now running.", "next": "Validate public cache/chunk flags, publish evidence and retain originalBF16 mismatches; RNGD long KONI queue remains active.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {}, "artifacts": [], "experiments": ["experiments/2026-10-23-gpu-live-pd240.md", "experiments/2026-10-23-pd-producer-cache241.md"], "labels": ["experiment"], "review": false, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-cli-qwen06b-http-module-failed.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-cli-qwen06b-http-module-failed.json new file mode 100644 index 00000000..8620ba7c --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-cli-qwen06b-http-module-failed.json @@ -0,0 +1,762 @@ +{ + "model_name": "c1899de289a04d12100db370d81485cdf75e47ca", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": false, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "bfloat16", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": "cuda:0", + "producer_prefix_cache": false, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 2.3326796669862233, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.9926271860022098, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 1.3373802119749598, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.3109587940271012, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.330016371968668, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 5.35376108600758, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.5297298840014264, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 1.4434433479909785, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 3.305800692003686, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 3.3507912129862234, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 3.8354775779880583, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 5.109192410018295, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 5.128718720981851, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 5.345180873002391, + "baseline_equal": true + } + ] + } + ], + "producer_chunk": 256, + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.465392636018805 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 0.9244075340102427 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.9534982640179805 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "error": "AssertionError()" +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-cli-qwen06b.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-cli-qwen06b.json new file mode 100644 index 00000000..d47dfdca --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-cli-qwen06b.json @@ -0,0 +1,777 @@ +{ + "model_name": "c1899de289a04d12100db370d81485cdf75e47ca", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "bfloat16", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": "cuda:0", + "producer_prefix_cache": false, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 2.0847649900242686, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.1438685829634778, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 1.0861156499595381, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.064916090981569, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.0818883000174537, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 4.337486725009512, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.9293278189725243, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 2.4829671480110846, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.148867590993177, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.178427481034305, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 2.7807450180407614, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 4.1220757070113905, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 4.140341215010267, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 4.329951549996622, + "baseline_equal": true + } + ] + } + ], + "producer_chunk": 256, + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.368116508994717 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 0.8909296370111406 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.9418938520248048 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.02\"} 2\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.04\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.06\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.08\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.25\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.75\"} 9\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 9\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 16\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 22.583908945031\nvllm:time_to_first_token_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.025\"} 11\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.05\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.46864654996334726\nvllm:time_per_output_token_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.8\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 6\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.5\"} 8\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.0\"} 9\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 15\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"5.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 37.11195199389476\nvllm:e2e_request_latency_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP airm_pd_prefills_total Prompts prefilled by the PD prefill worker (remote) or here (local).\n# TYPE airm_pd_prefills_total counter\nairm_pd_prefills_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",side=\"remote\"} 21\nairm_pd_prefills_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",side=\"local\"} 0\n# HELP airm_pd_transferred_tokens_total Prompt KV tokens received from the prefill side.\n# TYPE airm_pd_transferred_tokens_total counter\nairm_pd_transferred_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44544\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 485, + "prefill_chunks": 0, + "decode_steps": 411, + "decode_rows": 593, + "prefills": 21, + "transferred_tokens": 44544, + "decode_cached_tokens": 0, + "prefill_cached_tokens": 0, + "prefill_s": 6.719662656949367, + "transfer_s": 0.7056540118646808 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-native-fp32.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-native-fp32.json new file mode 100644 index 00000000..34d730e1 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-native-fp32.json @@ -0,0 +1,770 @@ +{ + "model_name": "Gemma3-1B", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "float32", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": null, + "producer_prefix_cache": null, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 3.9777029419783503, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 2.160653048020322, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 160068, + 531, + 506, + 2934, + 236764, + 1144, + 563, + 506 + ], + "e2e_s": 2.895906418969389, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 3.9758941539912485, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 2.992429861042183, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 7.989357012032997, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 2.206554489035625, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 160068, + 531, + 506, + 2934, + 236764, + 1144, + 563, + 506 + ], + "e2e_s": 2.983123419980984, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 4.4636899229954, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 4.5287682469934225, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 160068, + 531, + 506, + 2934, + 236764, + 1144, + 563, + 506 + ], + "e2e_s": 5.0777906340081245, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 7.789354478998575, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 5.147595846967306, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 160068, + 531, + 506, + 2934, + 236764, + 1144, + 563, + 506 + ], + "e2e_s": 7.974293201987166, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 12.125553819991183 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 160068, + 531, + 506, + 2934, + 236764, + 1144, + 563, + 506 + ], + "e2e_s": 1.067374253005255 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 2.6547895739786327 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "after_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "after_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Gemma3-1B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Gemma3-1B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Gemma3-1B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Gemma3-1B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Gemma3-1B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.04\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.25\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.75\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 12\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"5.0\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Gemma3-1B\"} 35.4343106248416\nvllm:time_to_first_token_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.025\"} 4\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.05\"} 13\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.075\"} 15\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.15\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.2\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.3\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Gemma3-1B\"} 1.1471506427098515\nvllm:time_per_output_token_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.8\"} 3\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.5\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.0\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 7\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"5.0\"} 14\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"10.0\"} 18\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Gemma3-1B\"} 70.995980548847\nvllm:e2e_request_latency_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Gemma3-1B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Gemma3-1B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Gemma3-1B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Gemma3-1B\"} 0\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 475, + "prefill_chunks": 180, + "decode_steps": 412, + "decode_rows": 594 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-native.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-native.json new file mode 100644 index 00000000..a940394b --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-native.json @@ -0,0 +1,770 @@ +{ + "model_name": "Gemma3-1B", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "bfloat16", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": null, + "producer_prefix_cache": null, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 3.0532956839888357, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 1.6318560150102712, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 236743, + 107, + 106, + 107, + 106, + 107, + 106, + 107 + ], + "e2e_s": 2.1016883910051547, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 3.051499545981642, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 2.164020080002956, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 5.723151587008033, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 1.6690447430009954, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 236743, + 107, + 106, + 107, + 106, + 107, + 106, + 107 + ], + "e2e_s": 2.1421356410137378, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 3.814490938966628, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 3.868307111028116, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 236743, + 107, + 106, + 107, + 106, + 107, + 106, + 107 + ], + "e2e_s": 4.342937940964475, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 5.446160580031574, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 5.475346184975933, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 236743, + 107, + 106, + 107, + 106, + 107, + 106, + 107 + ], + "e2e_s": 5.711868015001528, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 5.590034506982192 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 236743, + 107, + 106, + 107, + 106, + 107, + 106, + 107 + ], + "e2e_s": 1.079916720977053 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 1.766587783000432 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "after_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "after_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Gemma3-1B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Gemma3-1B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Gemma3-1B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Gemma3-1B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Gemma3-1B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.04\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.25\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.75\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 12\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 15\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Gemma3-1B\"} 26.44328100886196\nvllm:time_to_first_token_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.025\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.05\"} 10\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.075\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.15\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Gemma3-1B\"} 0.8616920843844362\nvllm:time_per_output_token_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.8\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.5\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.0\"} 8\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 11\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"5.0\"} 15\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Gemma3-1B\"} 53.15573562477948\nvllm:e2e_request_latency_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Gemma3-1B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Gemma3-1B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Gemma3-1B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Gemma3-1B\"} 0\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 455, + "prefill_chunks": 180, + "decode_steps": 412, + "decode_rows": 594 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-pd-cacheoff.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-pd-cacheoff.json new file mode 100644 index 00000000..d5634f60 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-pd-cacheoff.json @@ -0,0 +1,777 @@ +{ + "model_name": "Gemma3-1B", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "bfloat16", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": "cuda:0", + "producer_prefix_cache": false, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 1.2860346999950707, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 1.0738826440065168, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 115943, + 236890, + 106, + 237021, + 3110, + 21647, + 236830, + 26920 + ], + "e2e_s": 1.0421328120282851, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 1.2786466630059294, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 1.0928578660241328, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 2.3808348909951746, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 0.9808680249843746, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 115943, + 236890, + 106, + 237021, + 3110, + 21647, + 236830, + 26920 + ], + "e2e_s": 1.0838978019892238, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 1.3686736820382066, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 1.412662803020794, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 115943, + 236890, + 106, + 237021, + 3110, + 21647, + 236830, + 26920 + ], + "e2e_s": 2.096368213009555, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 2.2942207029554993, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 2.320422706019599, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 115943, + 236890, + 106, + 237021, + 3110, + 21647, + 236830, + 26920 + ], + "e2e_s": 2.3720040259649977, + "baseline_equal": true + } + ] + } + ], + "producer_chunk": 2048, + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 2.7645446800161153 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 115943, + 236890, + 106, + 237021, + 3110, + 21647, + 236830, + 26920 + ], + "e2e_s": 0.9360881990287453 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 1.1560799279832281 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "after_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "after_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Gemma3-1B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Gemma3-1B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Gemma3-1B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Gemma3-1B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Gemma3-1B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.04\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.06\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.08\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.25\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.75\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 15\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Gemma3-1B\"} 8.338732991949655\nvllm:time_to_first_token_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.025\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.05\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Gemma3-1B\"} 0.5916491781075995\nvllm:time_per_output_token_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.8\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 6\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.5\"} 14\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.0\"} 14\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 18\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"5.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Gemma3-1B\"} 26.679857513285242\nvllm:e2e_request_latency_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Gemma3-1B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Gemma3-1B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Gemma3-1B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Gemma3-1B\"} 0\n# HELP airm_pd_prefills_total Prompts prefilled by the PD prefill worker (remote) or here (local).\n# TYPE airm_pd_prefills_total counter\nairm_pd_prefills_total{model_name=\"Gemma3-1B\",side=\"remote\"} 21\nairm_pd_prefills_total{model_name=\"Gemma3-1B\",side=\"local\"} 0\n# HELP airm_pd_transferred_tokens_total Prompt KV tokens received from the prefill side.\n# TYPE airm_pd_transferred_tokens_total counter\nairm_pd_transferred_tokens_total{model_name=\"Gemma3-1B\"} 44544\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 361, + "prefill_chunks": 0, + "decode_steps": 337, + "decode_rows": 594, + "prefills": 21, + "transferred_tokens": 44544, + "decode_cached_tokens": 0, + "prefill_cached_tokens": 0, + "prefill_s": 2.3563580660847947, + "transfer_s": 0.19421064283233136 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-pd-chunk256.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-pd-chunk256.json new file mode 100644 index 00000000..529e5148 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-pd-chunk256.json @@ -0,0 +1,777 @@ +{ + "model_name": "Gemma3-1B", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "bfloat16", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": "cuda:0", + "producer_prefix_cache": false, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 3.018663992988877, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 1.450199261016678, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 236743, + 107, + 106, + 107, + 106, + 107, + 106, + 107 + ], + "e2e_s": 2.0216126950108446, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 2.9894438359769993, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 3.015440598013811, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 6.302969884010963, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 1.7112768180086277, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 236743, + 107, + 106, + 107, + 106, + 107, + 106, + 107 + ], + "e2e_s": 2.248694477020763, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 4.0818270379677415, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 4.19195549399592, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 236743, + 107, + 106, + 107, + 106, + 107, + 106, + 107 + ], + "e2e_s": 4.634602417005226, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 6.046006623015273, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 6.09167690196773, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 236743, + 107, + 106, + 107, + 106, + 107, + 106, + 107 + ], + "e2e_s": 6.291903668025043, + "baseline_equal": true + } + ] + } + ], + "producer_chunk": 256, + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 2.591466554964427 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 236743, + 107, + 106, + 107, + 106, + 107, + 106, + 107 + ], + "e2e_s": 0.9895597780123353 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 1.6942639049957506 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "after_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "after_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Gemma3-1B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Gemma3-1B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Gemma3-1B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Gemma3-1B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Gemma3-1B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.04\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.25\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.75\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 15\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"5.0\"} 18\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Gemma3-1B\"} 30.05537067214027\nvllm:time_to_first_token_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.025\"} 3\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.05\"} 12\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Gemma3-1B\"} 0.7474881432229474\nvllm:time_per_output_token_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.8\"} 2\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.5\"} 6\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.0\"} 8\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 10\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"5.0\"} 16\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Gemma3-1B\"} 53.227503112051636\nvllm:e2e_request_latency_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Gemma3-1B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Gemma3-1B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Gemma3-1B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Gemma3-1B\"} 0\n# HELP airm_pd_prefills_total Prompts prefilled by the PD prefill worker (remote) or here (local).\n# TYPE airm_pd_prefills_total counter\nairm_pd_prefills_total{model_name=\"Gemma3-1B\",side=\"remote\"} 21\nairm_pd_prefills_total{model_name=\"Gemma3-1B\",side=\"local\"} 0\n# HELP airm_pd_transferred_tokens_total Prompt KV tokens received from the prefill side.\n# TYPE airm_pd_transferred_tokens_total counter\nairm_pd_transferred_tokens_total{model_name=\"Gemma3-1B\"} 44544\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 461, + "prefill_chunks": 0, + "decode_steps": 418, + "decode_rows": 594, + "prefills": 21, + "transferred_tokens": 44544, + "decode_cached_tokens": 0, + "prefill_cached_tokens": 0, + "prefill_s": 9.238624927180354, + "transfer_s": 0.1933901718002744 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-pd-fp32.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-pd-fp32.json new file mode 100644 index 00000000..e85e3c47 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-pd-fp32.json @@ -0,0 +1,777 @@ +{ + "model_name": "Gemma3-1B", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "float32", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": "cuda:0", + "producer_prefix_cache": true, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 1.2166953529813327, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 1.107358306995593, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 160068, + 531, + 506, + 2934, + 236764, + 1144, + 563, + 506 + ], + "e2e_s": 1.0515554849989712, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 1.215171218966134, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 1.1320417110109702, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 2.1551846799557097, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 0.9583865639870055, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 160068, + 531, + 506, + 2934, + 236764, + 1144, + 563, + 506 + ], + "e2e_s": 1.0675180690013804, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 1.1636857789708301, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 1.204473999037873, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 160068, + 531, + 506, + 2934, + 236764, + 1144, + 563, + 506 + ], + "e2e_s": 1.956552563002333, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 2.0723104610224254, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 2.1212833340396173, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 160068, + 531, + 506, + 2934, + 236764, + 1144, + 563, + 506 + ], + "e2e_s": 2.149569465953391, + "baseline_equal": true + } + ] + } + ], + "producer_chunk": 2048, + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 2.8043884730432183 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 160068, + 531, + 506, + 2934, + 236764, + 1144, + 563, + 506 + ], + "e2e_s": 1.0543557520140894 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 1.8492919130367227 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "after_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "after_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Gemma3-1B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Gemma3-1B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Gemma3-1B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Gemma3-1B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Gemma3-1B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.04\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.06\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.25\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 13\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.75\"} 13\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 15\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Gemma3-1B\"} 8.261962681892328\nvllm:time_to_first_token_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.025\"} 2\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.05\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Gemma3-1B\"} 0.5732024040658988\nvllm:time_per_output_token_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.8\"} 1\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.5\"} 13\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.0\"} 15\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 18\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"5.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Gemma3-1B\"} 26.03123720793519\nvllm:e2e_request_latency_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Gemma3-1B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Gemma3-1B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Gemma3-1B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Gemma3-1B\"} 0\n# HELP airm_pd_prefills_total Prompts prefilled by the PD prefill worker (remote) or here (local).\n# TYPE airm_pd_prefills_total counter\nairm_pd_prefills_total{model_name=\"Gemma3-1B\",side=\"remote\"} 21\nairm_pd_prefills_total{model_name=\"Gemma3-1B\",side=\"local\"} 0\n# HELP airm_pd_transferred_tokens_total Prompt KV tokens received from the prefill side.\n# TYPE airm_pd_transferred_tokens_total counter\nairm_pd_transferred_tokens_total{model_name=\"Gemma3-1B\"} 44544\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 371, + "prefill_chunks": 0, + "decode_steps": 330, + "decode_rows": 594, + "prefills": 21, + "transferred_tokens": 44544, + "decode_cached_tokens": 0, + "prefill_cached_tokens": 35072, + "prefill_s": 2.580388412810862, + "transfer_s": 0.37793712719576433 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-pd.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-pd.json new file mode 100644 index 00000000..5ab3901e --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-gemma-pd.json @@ -0,0 +1,776 @@ +{ + "model_name": "Gemma3-1B", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "bfloat16", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": "cuda:0", + "producer_prefix_cache": true, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 1.099843726027757, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 0.9667452269932255, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 160068, + 531, + 506, + 2934, + 236764, + 1144, + 563, + 506 + ], + "e2e_s": 1.0456489339703694, + "baseline_equal": false + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 1.0732492789975367, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 1.097403429041151, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 1.9888289099908434, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 1.005784616980236, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 160068, + 531, + 506, + 2934, + 236764, + 1144, + 563, + 506 + ], + "e2e_s": 0.8905859780497849, + "baseline_equal": false + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 1.8553098670090549, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 1.091579636966344, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 160068, + 531, + 506, + 2934, + 236764, + 1144, + 563, + 506 + ], + "e2e_s": 1.0492946419981308, + "baseline_equal": false + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 1.9286614619777538, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 1.9556639269576408, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 160068, + 531, + 506, + 2934, + 236764, + 1144, + 563, + 506 + ], + "e2e_s": 1.9819010009523481, + "baseline_equal": false + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 2.684294531994965 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 115943, + 236890, + 106, + 237021, + 3110, + 21647, + 236830, + 26920 + ], + "e2e_s": 0.9034020059625618 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 1.2361804000101984 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "after_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "after_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Gemma3-1B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Gemma3-1B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Gemma3-1B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Gemma3-1B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Gemma3-1B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.04\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.08\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.25\"} 12\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.75\"} 15\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 16\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Gemma3-1B\"} 6.849860136106145\nvllm:time_to_first_token_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.025\"} 1\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.05\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Gemma3-1B\"} 0.5509548555823012\nvllm:time_per_output_token_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.8\"} 1\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 8\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.5\"} 14\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.0\"} 18\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 18\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"5.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Gemma3-1B\"} 23.929460659157485\nvllm:e2e_request_latency_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Gemma3-1B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Gemma3-1B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Gemma3-1B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Gemma3-1B\"} 0\n# HELP airm_pd_prefills_total Prompts prefilled by the PD prefill worker (remote) or here (local).\n# TYPE airm_pd_prefills_total counter\nairm_pd_prefills_total{model_name=\"Gemma3-1B\",side=\"remote\"} 21\nairm_pd_prefills_total{model_name=\"Gemma3-1B\",side=\"local\"} 0\n# HELP airm_pd_transferred_tokens_total Prompt KV tokens received from the prefill side.\n# TYPE airm_pd_transferred_tokens_total counter\nairm_pd_transferred_tokens_total{model_name=\"Gemma3-1B\"} 44544\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 351, + "prefill_chunks": 0, + "decode_steps": 327, + "decode_rows": 594, + "prefills": 21, + "transferred_tokens": 44544, + "decode_cached_tokens": 0, + "prefill_cached_tokens": 35072, + "prefill_s": 1.7415074450545944, + "transfer_s": 0.19139915297273546 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-koni-native-fp32.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-koni-native-fp32.json new file mode 100644 index 00000000..b24a0d2a --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-koni-native-fp32.json @@ -0,0 +1,770 @@ +{ + "model_name": "KONI-8B", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "float32", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": null, + "producer_prefix_cache": null, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 14.902697762998287, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 9.337801988003775, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 12.234848832013085, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 14.900826631987002, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 12.606564885005355, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 30.82044083904475, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 11.719484357046895, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 11.366077370010316, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 25.538585569011047, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 25.912523662962485, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 14.649988512974232, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 30.459919543005526, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 30.499071028025355, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 30.814749545999803, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 5.655489575001411 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 3.2816724309814163 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 10.989560924004763 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "after_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "after_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"KONI-8B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"KONI-8B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"KONI-8B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"KONI-8B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"KONI-8B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"KONI-8B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"KONI-8B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"KONI-8B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.04\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.06\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.08\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.25\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.75\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"5.0\"} 12\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"7.5\"} 12\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"10.0\"} 13\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"KONI-8B\"} 153.71604993316578\nvllm:time_to_first_token_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.025\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.05\"} 6\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.075\"} 8\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 8\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.15\"} 10\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.2\"} 11\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.3\"} 13\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"KONI-8B\"} 3.262645549580245\nvllm:time_per_output_token_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.8\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"1.5\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"2.0\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"5.0\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"10.0\"} 7\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"15.0\"} 14\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"20.0\"} 14\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"30.0\"} 16\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"KONI-8B\"} 254.85806197015336\nvllm:e2e_request_latency_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"KONI-8B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"KONI-8B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"KONI-8B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"KONI-8B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"KONI-8B\"} 0\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 454, + "prefill_chunks": 180, + "decode_steps": 404, + "decode_rows": 593 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-koni-native.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-koni-native.json new file mode 100644 index 00000000..74d13bf5 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-koni-native.json @@ -0,0 +1,770 @@ +{ + "model_name": "KONI-8B", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "bfloat16", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": null, + "producer_prefix_cache": null, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 3.7856353640090674, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "e2e_s": 2.1550977349979803, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 2.903286832035519, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 3.762725483975373, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "e2e_s": 3.7820582190179266, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 7.4094623170094565, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "e2e_s": 2.1496002160129137, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 2.899106347002089, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 5.419189491018187, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "e2e_s": 5.517913390998729, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 7.220115938980598, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 7.0563988870126195, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "e2e_s": 7.240670682978816, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 7.4046842670068145, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "e2e_s": 2.4633299779961817 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 1.0198347619734704 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 2.592131611017976 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "after_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "after_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"KONI-8B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"KONI-8B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"KONI-8B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"KONI-8B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"KONI-8B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"KONI-8B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"KONI-8B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"KONI-8B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.04\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.25\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.75\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"5.0\"} 15\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"KONI-8B\"} 41.35759315191535\nvllm:time_to_first_token_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.025\"} 9\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.05\"} 12\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.075\"} 15\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"KONI-8B\"} 0.8017549140664239\nvllm:time_per_output_token_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.8\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"1.5\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"2.0\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 8\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"5.0\"} 13\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"KONI-8B\"} 66.21199548797449\nvllm:e2e_request_latency_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"KONI-8B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"KONI-8B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"KONI-8B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"KONI-8B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"KONI-8B\"} 0\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 454, + "prefill_chunks": 180, + "decode_steps": 411, + "decode_rows": 593 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-koni-pd-cacheoff.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-koni-pd-cacheoff.json new file mode 100644 index 00000000..e2425ac6 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-koni-pd-cacheoff.json @@ -0,0 +1,777 @@ +{ + "model_name": "KONI-8B", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "bfloat16", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": "cuda:0", + "producer_prefix_cache": false, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 2.043118086992763, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "e2e_s": 0.7736373239895329, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 1.0147145119844936, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 2.0194858700269833, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "e2e_s": 2.041229797003325, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 4.03540851798607, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "e2e_s": 0.7284507830045186, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 2.0646306379931048, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 1.8640459070447832, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "e2e_s": 2.0958602459868416, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 2.32417511503445, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 3.8529659509658813, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "e2e_s": 2.633719688979909, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 4.026027096027974, + "baseline_equal": true + } + ] + } + ], + "producer_chunk": 2048, + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "e2e_s": 2.599120829021558 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 0.904170672991313 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 2.0787297069909982 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "after_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "after_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"KONI-8B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"KONI-8B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"KONI-8B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"KONI-8B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"KONI-8B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"KONI-8B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"KONI-8B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"KONI-8B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.04\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.25\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.75\"} 9\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 9\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"KONI-8B\"} 19.55902083090041\nvllm:time_to_first_token_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.025\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.05\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"KONI-8B\"} 0.4498017242257182\nvllm:time_per_output_token_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.8\"} 6\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 7\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"1.5\"} 8\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"2.0\"} 9\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 15\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"5.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"KONI-8B\"} 33.502874281897675\nvllm:e2e_request_latency_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"KONI-8B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"KONI-8B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"KONI-8B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"KONI-8B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"KONI-8B\"} 0\n# HELP airm_pd_prefills_total Prompts prefilled by the PD prefill worker (remote) or here (local).\n# TYPE airm_pd_prefills_total counter\nairm_pd_prefills_total{model_name=\"KONI-8B\",side=\"remote\"} 21\nairm_pd_prefills_total{model_name=\"KONI-8B\",side=\"local\"} 0\n# HELP airm_pd_transferred_tokens_total Prompt KV tokens received from the prefill side.\n# TYPE airm_pd_transferred_tokens_total counter\nairm_pd_transferred_tokens_total{model_name=\"KONI-8B\"} 44544\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 502, + "prefill_chunks": 0, + "decode_steps": 429, + "decode_rows": 593, + "prefills": 21, + "transferred_tokens": 44544, + "decode_cached_tokens": 0, + "prefill_cached_tokens": 0, + "prefill_s": 6.09040836483473, + "transfer_s": 0.7627043939428404 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-koni-pd-fp32.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-koni-pd-fp32.json new file mode 100644 index 00000000..dd0dd673 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-koni-pd-fp32.json @@ -0,0 +1,777 @@ +{ + "model_name": "KONI-8B", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "float32", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": "cuda:0", + "producer_prefix_cache": true, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 2.2754032199736685, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 1.803892811993137, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 1.6455628999974579, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 2.2739371380303055, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 1.9068794659688137, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 4.0792744490318, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 1.5789754450088367, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 1.7974969970528036, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 2.295974743028637, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 1.9041497090365738, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 3.4351436950382777, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 3.9169364440022036, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 3.5940502189914696, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 4.071274012036156, + "baseline_equal": true + } + ] + } + ], + "producer_chunk": 2048, + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 3.1298927160096355 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 3.182248790981248 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 8.47563371900469 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "after_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "after_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"KONI-8B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"KONI-8B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"KONI-8B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"KONI-8B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"KONI-8B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"KONI-8B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"KONI-8B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"KONI-8B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.04\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.06\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.08\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.25\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.75\"} 12\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 13\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 18\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"5.0\"} 18\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"KONI-8B\"} 20.96457086794544\nvllm:time_to_first_token_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.025\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.05\"} 8\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"KONI-8B\"} 0.9272468037125954\nvllm:time_per_output_token_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.8\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"1.5\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"2.0\"} 10\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 12\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"5.0\"} 18\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"KONI-8B\"} 49.7092217830359\nvllm:e2e_request_latency_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"KONI-8B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"KONI-8B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"KONI-8B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"KONI-8B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"KONI-8B\"} 0\n# HELP airm_pd_prefills_total Prompts prefilled by the PD prefill worker (remote) or here (local).\n# TYPE airm_pd_prefills_total counter\nairm_pd_prefills_total{model_name=\"KONI-8B\",side=\"remote\"} 21\nairm_pd_prefills_total{model_name=\"KONI-8B\",side=\"local\"} 0\n# HELP airm_pd_transferred_tokens_total Prompt KV tokens received from the prefill side.\n# TYPE airm_pd_transferred_tokens_total counter\nairm_pd_transferred_tokens_total{model_name=\"KONI-8B\"} 44544\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 542, + "prefill_chunks": 0, + "decode_steps": 342, + "decode_rows": 593, + "prefills": 21, + "transferred_tokens": 44544, + "decode_cached_tokens": 0, + "prefill_cached_tokens": 35072, + "prefill_s": 10.498879219812807, + "transfer_s": 1.501304489269387 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-koni-pd.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-koni-pd.json new file mode 100644 index 00000000..128ee687 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-koni-pd.json @@ -0,0 +1,776 @@ +{ + "model_name": "KONI-8B", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "bfloat16", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": "cuda:0", + "producer_prefix_cache": true, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 0.9576541220303625, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 4495, + 30, + 7566, + 627, + 15546, + 374, + 279, + 4900, + 315, + 9822, + 5380, + 16533, + 25, + 578, + 4900, + 315, + 9822, + 374, + 61544, + 63401, + 627 + ], + "e2e_s": 0.7447484289878048, + "baseline_equal": false + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 0.8307239079731517, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 0.9351009399979375, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 4495, + 30, + 7566, + 627, + 15546, + 374, + 279, + 4900, + 315, + 9822, + 5380, + 16533, + 25, + 578, + 4900, + 315, + 9822, + 374, + 61544, + 63401, + 627 + ], + "e2e_s": 0.9554941349779256, + "baseline_equal": false + } + ] + }, + { + "concurrency": 8, + "wall_s": 1.8048956899438053, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 4495, + 30, + 7566, + 627, + 15546, + 374, + 279, + 4900, + 315, + 9822, + 5380, + 16533, + 25, + 578, + 4900, + 315, + 9822, + 374, + 61544, + 63401, + 627 + ], + "e2e_s": 0.7823926020064391, + "baseline_equal": false + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 0.8776985470321961, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 1.0150665560504422, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 4495, + 30, + 7566, + 627, + 15546, + 374, + 279, + 4900, + 315, + 9822, + 5380, + 16533, + 25, + 578, + 4900, + 315, + 9822, + 374, + 61544, + 63401, + 627 + ], + "e2e_s": 1.046619406028185, + "baseline_equal": false + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 1.6104507009731606, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 1.7370283570489846, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 4495, + 30, + 7566, + 627, + 15546, + 374, + 279, + 4900, + 315, + 9822, + 5380, + 16533, + 25, + 578, + 4900, + 315, + 9822, + 374, + 61544, + 63401, + 627 + ], + "e2e_s": 1.8005936730187386, + "baseline_equal": false + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 1.7554664550116286, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 779, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 617, + 904, + 1023, + 4860, + 13, + 720, + 38766, + 1303, + 33025, + 2696 + ], + "e2e_s": 2.5390587460133247 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 0.9969039050047286 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 1.9841613399912603 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": false, + "before_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 4495, + 30, + 7566, + 627, + 15546, + 374, + 279, + 4900, + 315, + 9822, + 5380, + 16533, + 25, + 578, + 4900, + 315, + 9822, + 374, + 61544, + 63401, + 627 + ], + "after_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 4495, + 30, + 7566, + 627, + 15546, + 374, + 279, + 4900, + 315, + 9822, + 5380, + 16533, + 25, + 578, + 4900, + 315, + 9822, + 374, + 61544, + 63401, + 627 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": false, + "before_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 4495, + 30, + 7566, + 627, + 15546, + 374, + 279, + 4900, + 315, + 9822, + 5380, + 16533, + 25, + 578, + 4900, + 315, + 9822, + 374, + 61544, + 63401, + 627 + ], + "after_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 4495, + 30, + 7566, + 627, + 15546, + 374, + 279, + 4900, + 315, + 9822, + 5380, + 16533, + 25, + 578, + 4900, + 315, + 9822, + 374, + 61544, + 63401, + 627 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"KONI-8B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"KONI-8B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"KONI-8B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"KONI-8B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"KONI-8B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"KONI-8B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"KONI-8B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"KONI-8B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.04\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.25\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 13\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.75\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 15\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"KONI-8B\"} 7.979920896992553\nvllm:time_to_first_token_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.025\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.05\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"KONI-8B\"} 0.4580228750654045\nvllm:time_per_output_token_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.8\"} 6\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 11\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"1.5\"} 13\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"2.0\"} 18\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 18\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"5.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"KONI-8B\"} 22.178630024020094\nvllm:e2e_request_latency_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"KONI-8B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"KONI-8B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"KONI-8B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"KONI-8B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"KONI-8B\"} 0\n# HELP airm_pd_prefills_total Prompts prefilled by the PD prefill worker (remote) or here (local).\n# TYPE airm_pd_prefills_total counter\nairm_pd_prefills_total{model_name=\"KONI-8B\",side=\"remote\"} 21\nairm_pd_prefills_total{model_name=\"KONI-8B\",side=\"local\"} 0\n# HELP airm_pd_transferred_tokens_total Prompt KV tokens received from the prefill side.\n# TYPE airm_pd_transferred_tokens_total counter\nairm_pd_transferred_tokens_total{model_name=\"KONI-8B\"} 44544\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 384, + "prefill_chunks": 0, + "decode_steps": 336, + "decode_rows": 593, + "prefills": 21, + "transferred_tokens": 44544, + "decode_cached_tokens": 0, + "prefill_cached_tokens": 35072, + "prefill_s": 2.566409863065928, + "transfer_s": 0.7837857630802318 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-native-fp32.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-native-fp32.json new file mode 100644 index 00000000..d1caf7af --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-native-fp32.json @@ -0,0 +1,770 @@ +{ + "model_name": "c1899de289a04d12100db370d81485cdf75e47ca", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "float32", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": null, + "producer_prefix_cache": null, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 3.5071813319809735, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.497774371993728, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 2.403283718973398, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 3.4874001209973358, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 3.5045639610034414, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 6.984758931968827, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.469293050991837, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 2.37511706299847, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 4.742106474004686, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 4.821220875019208, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 5.5236580750206485, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 6.82445378502598, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 6.842242162034381, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 6.975589729030617, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.3574264150229283 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 0.8609134260332212 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.5949818599619903 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.04\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.06\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.08\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.25\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 9\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.75\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"5.0\"} 16\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 35.53010373894358\nvllm:time_to_first_token_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.025\"} 6\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.05\"} 11\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.075\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.805575875095993\nvllm:time_per_output_token_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.8\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.5\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.0\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 10\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"5.0\"} 15\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 60.502955866919365\nvllm:e2e_request_latency_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 454, + "prefill_chunks": 180, + "decode_steps": 397, + "decode_rows": 593 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-native.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-native.json new file mode 100644 index 00000000..9fd80286 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-native.json @@ -0,0 +1,770 @@ +{ + "model_name": "c1899de289a04d12100db370d81485cdf75e47ca", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "bfloat16", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": null, + "producer_prefix_cache": null, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 2.431407799012959, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.6773576259729452, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 1.614428402041085, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.430236171989236, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.7372710650088266, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 4.538768605969381, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.6529965489753522, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 1.5903503029840067, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 3.0005052710184827, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 3.5180881539708935, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 3.451193240005523, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 4.352662532997783, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 4.3712557179969735, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 4.529504529957194, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.4313674430013634 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 0.8107867439975962 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.582710599002894 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.02\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.04\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.06\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.08\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.25\"} 9\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.75\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 12\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 16\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 21.796314835897647\nvllm:time_to_first_token_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.025\"} 10\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.05\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.6243327037119607\nvllm:time_per_output_token_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.8\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.5\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.0\"} 11\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 13\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"5.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 41.15062865096843\nvllm:e2e_request_latency_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 454, + "prefill_chunks": 180, + "decode_steps": 397, + "decode_rows": 593 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-pd-cacheoff.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-pd-cacheoff.json new file mode 100644 index 00000000..45059ec1 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-pd-cacheoff.json @@ -0,0 +1,777 @@ +{ + "model_name": "c1899de289a04d12100db370d81485cdf75e47ca", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "bfloat16", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": "cuda:0", + "producer_prefix_cache": false, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 1.2276619570329785, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.8229152209823951, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 0.8750991469714791, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.2007423600298353, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.2239311860175803, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 1.964382121048402, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.8627434269874357, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 0.7997505219536833, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.2411072389804758, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.2755011080298573, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 1.6130367319565266, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.8967565320199355, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.9147166849579662, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 1.9530336200259626, + "baseline_equal": true + } + ] + } + ], + "producer_chunk": 2048, + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.4406586210243404 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 0.7707657489809208 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.3600870580412447 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.04\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.06\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.25\"} 9\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 12\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.75\"} 15\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 16\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 8.594117011118215\nvllm:time_to_first_token_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.025\"} 16\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.05\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.4573455693560741\nvllm:time_per_output_token_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.8\"} 6\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 9\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.5\"} 14\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.0\"} 18\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"5.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 22.771829661156517\nvllm:e2e_request_latency_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP airm_pd_prefills_total Prompts prefilled by the PD prefill worker (remote) or here (local).\n# TYPE airm_pd_prefills_total counter\nairm_pd_prefills_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",side=\"remote\"} 21\nairm_pd_prefills_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",side=\"local\"} 0\n# HELP airm_pd_transferred_tokens_total Prompt KV tokens received from the prefill side.\n# TYPE airm_pd_transferred_tokens_total counter\nairm_pd_transferred_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44544\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 382, + "prefill_chunks": 0, + "decode_steps": 350, + "decode_rows": 593, + "prefills": 21, + "transferred_tokens": 44544, + "decode_cached_tokens": 0, + "prefill_cached_tokens": 0, + "prefill_s": 2.4357351638609543, + "transfer_s": 0.7212044671177864 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-pd-chunk256.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-pd-chunk256.json new file mode 100644 index 00000000..ac5e67a8 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-pd-chunk256.json @@ -0,0 +1,777 @@ +{ + "model_name": "c1899de289a04d12100db370d81485cdf75e47ca", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "bfloat16", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": "cuda:0", + "producer_prefix_cache": false, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 2.8673682369990274, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.3028777390136383, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 1.8626208530040458, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.8657870809547603, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.8616604970302433, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 5.760858462017495, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.4187441089889035, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 3.4285738909966312, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.9404824909870513, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 3.4768392390105873, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 3.982198995014187, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 5.505825863976497, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 5.527052438992541, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 5.753948945028242, + "baseline_equal": true + } + ] + } + ], + "producer_chunk": 256, + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.5558548089466058 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 0.9665324870147742 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.019523546041455 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.04\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.25\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 9\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.75\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 15\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"5.0\"} 18\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 29.04814301506849\nvllm:time_to_first_token_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.025\"} 8\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.05\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.611529255060539\nvllm:time_per_output_token_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.8\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.5\"} 7\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.0\"} 9\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 10\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"5.0\"} 16\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 48.0055499219452\nvllm:e2e_request_latency_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP airm_pd_prefills_total Prompts prefilled by the PD prefill worker (remote) or here (local).\n# TYPE airm_pd_prefills_total counter\nairm_pd_prefills_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",side=\"remote\"} 21\nairm_pd_prefills_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",side=\"local\"} 0\n# HELP airm_pd_transferred_tokens_total Prompt KV tokens received from the prefill side.\n# TYPE airm_pd_transferred_tokens_total counter\nairm_pd_transferred_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44544\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 496, + "prefill_chunks": 0, + "decode_steps": 423, + "decode_rows": 593, + "prefills": 21, + "transferred_tokens": 44544, + "decode_cached_tokens": 0, + "prefill_cached_tokens": 0, + "prefill_s": 8.78918537311256, + "transfer_s": 0.7344166348339058 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-pd-fp32.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-pd-fp32.json new file mode 100644 index 00000000..f67df17f --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-pd-fp32.json @@ -0,0 +1,777 @@ +{ + "model_name": "c1899de289a04d12100db370d81485cdf75e47ca", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "float32", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": "cuda:0", + "producer_prefix_cache": true, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 1.211037187022157, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.8430193120148033, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 0.9532264560111798, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.1891977630439214, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.208066294027958, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 2.1782777379848994, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.8589822000358254, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 0.9788742859964259, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.21663473197259, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.2791375360102393, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 1.8378525230218656, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.056237449985929, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.1133575619896874, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 2.1698830280220136, + "baseline_equal": true + } + ] + } + ], + "producer_chunk": 2048, + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 5.372313789965119 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 0.9685679909889586 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.4987345300032757 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.04\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.25\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 13\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.75\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 15\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 9.349102522071917\nvllm:time_to_first_token_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.025\"} 5\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.05\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.075\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.5977893955160624\nvllm:time_per_output_token_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.8\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 9\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.5\"} 13\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.0\"} 14\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 18\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"5.0\"} 18\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 27.88057378306985\nvllm:e2e_request_latency_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP airm_pd_prefills_total Prompts prefilled by the PD prefill worker (remote) or here (local).\n# TYPE airm_pd_prefills_total counter\nairm_pd_prefills_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",side=\"remote\"} 21\nairm_pd_prefills_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",side=\"local\"} 0\n# HELP airm_pd_transferred_tokens_total Prompt KV tokens received from the prefill side.\n# TYPE airm_pd_transferred_tokens_total counter\nairm_pd_transferred_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44544\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 401, + "prefill_chunks": 0, + "decode_steps": 345, + "decode_rows": 593, + "prefills": 21, + "transferred_tokens": 44544, + "decode_cached_tokens": 0, + "prefill_cached_tokens": 35072, + "prefill_s": 2.477933837159071, + "transfer_s": 1.4616268919198774 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-pd.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-pd.json new file mode 100644 index 00000000..43c40fe3 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen06b-pd.json @@ -0,0 +1,776 @@ +{ + "model_name": "c1899de289a04d12100db370d81485cdf75e47ca", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "bfloat16", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": "cuda:0", + "producer_prefix_cache": true, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 0.8813359650084749, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.7186987139866687, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 0.7586602519731969, + "baseline_equal": false + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.858193815976847, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.8786629929672927, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 1.671834030013997, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.8014710430288687, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 0.7457747369771823, + "baseline_equal": false + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.9062358910450712, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.4994292149785906, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 0.9641078630229458, + "baseline_equal": false + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.6026211199932732, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.6236728960066102, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 1.6659170460188761, + "baseline_equal": false + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 4.2560367549886 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 0.7710132359643467 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.3248224830022082 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.04\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.25\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 13\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.75\"} 15\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 18\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 6.620681920088828\nvllm:time_to_first_token_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.025\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.05\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.075\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.49102969535235913\nvllm:time_per_output_token_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.8\"} 9\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 13\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.5\"} 15\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.0\"} 18\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"2.5\"} 18\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"5.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 21.84260247601196\nvllm:e2e_request_latency_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 0\n# HELP airm_pd_prefills_total Prompts prefilled by the PD prefill worker (remote) or here (local).\n# TYPE airm_pd_prefills_total counter\nairm_pd_prefills_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",side=\"remote\"} 21\nairm_pd_prefills_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\",side=\"local\"} 0\n# HELP airm_pd_transferred_tokens_total Prompt KV tokens received from the prefill side.\n# TYPE airm_pd_transferred_tokens_total counter\nairm_pd_transferred_tokens_total{model_name=\"c1899de289a04d12100db370d81485cdf75e47ca\"} 44544\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 364, + "prefill_chunks": 0, + "decode_steps": 332, + "decode_rows": 593, + "prefills": 21, + "transferred_tokens": 44544, + "decode_cached_tokens": 0, + "prefill_cached_tokens": 35072, + "prefill_s": 1.7282502409652807, + "transfer_s": 0.7161287839408033 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen4b-native.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen4b-native.json new file mode 100644 index 00000000..def4d7bd --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen4b-native.json @@ -0,0 +1,770 @@ +{ + "model_name": "Qwen3-4B", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "bfloat16", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": null, + "producer_prefix_cache": null, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 3.8185519209946506, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 2.0162408080068417, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 2.763667438994162, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 3.817076214996632, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 2.861940663016867, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 7.298394410987385, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 2.1180661359685473, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 2.858919277030509, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 5.020310607040301, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 5.10625822894508, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 5.849223237019032, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 7.078244542004541, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 7.101729045971297, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 7.292196044989396, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 4.1256311609758995 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 1.0856754249543883 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 2.469575277005788 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "after_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "after_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Qwen3-4B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Qwen3-4B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Qwen3-4B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Qwen3-4B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Qwen3-4B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.04\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.25\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.75\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 12\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"5.0\"} 16\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Qwen3-4B\"} 35.50801407295512\nvllm:time_to_first_token_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.025\"} 4\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.05\"} 10\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.075\"} 16\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Qwen3-4B\"} 0.9403607867444084\nvllm:time_per_output_token_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.8\"} 2\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.5\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.0\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 8\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"5.0\"} 13\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Qwen3-4B\"} 64.65919846203178\nvllm:e2e_request_latency_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Qwen3-4B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Qwen3-4B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Qwen3-4B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Qwen3-4B\"} 0\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 454, + "prefill_chunks": 180, + "decode_steps": 411, + "decode_rows": 593 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen4b-pd.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen4b-pd.json new file mode 100644 index 00000000..6cc78a81 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-qwen4b-pd.json @@ -0,0 +1,776 @@ +{ + "model_name": "Qwen3-4B", + "prefix_cache": false, + "host_kv_gib": 0, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.13.0+cu130", + "dtype": "bfloat16", + "backend": "torch", + "decode_device": "cuda:1", + "pd_prefill_device": "cuda:0", + "producer_prefix_cache": true, + "slots": 4, + "atom_kv_blocks": null, + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 1.1373623789986596, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 0.995525564008858, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 0.9411897380487062, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 1.1359665850177407, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 1.0241573519888334, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 2.2715368989738636, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 0.9257064529811032, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 1.0289316299604252, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 2.1082855139975436, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 1.1037134979851544, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 1.1349444990046322, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 2.213716446014587, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 1.9433539389865473, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 2.2664776009623893, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 2.746489177981857 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 1.2019724039710127 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 2.3052625579875894 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "after_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "after_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Qwen3-4B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Qwen3-4B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Qwen3-4B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Qwen3-4B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Qwen3-4B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.04\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.25\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 13\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.75\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 15\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Qwen3-4B\"} 8.96585211402271\nvllm:time_to_first_token_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.025\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.05\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Qwen3-4B\"} 0.5623500477710378\nvllm:time_per_output_token_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.8\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 7\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.5\"} 13\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.0\"} 14\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 18\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"5.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Qwen3-4B\"} 26.398703594924882\nvllm:e2e_request_latency_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Qwen3-4B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Qwen3-4B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Qwen3-4B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Qwen3-4B\"} 0\n# HELP airm_pd_prefills_total Prompts prefilled by the PD prefill worker (remote) or here (local).\n# TYPE airm_pd_prefills_total counter\nairm_pd_prefills_total{model_name=\"Qwen3-4B\",side=\"remote\"} 21\nairm_pd_prefills_total{model_name=\"Qwen3-4B\",side=\"local\"} 0\n# HELP airm_pd_transferred_tokens_total Prompt KV tokens received from the prefill side.\n# TYPE airm_pd_transferred_tokens_total counter\nairm_pd_transferred_tokens_total{model_name=\"Qwen3-4B\"} 44544\n", + "prefix_hit_tokens": 0.0, + "batcher_stats": { + "iterations": 387, + "prefill_chunks": 0, + "decode_steps": 335, + "decode_rows": 593, + "prefills": 21, + "transferred_tokens": 44544, + "decode_cached_tokens": 0, + "prefill_cached_tokens": 35072, + "prefill_s": 2.436243576987181, + "transfer_s": 1.2161275959806517 + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence240/gpu240-summary.json b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-summary.json new file mode 100644 index 00000000..30073f16 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence240/gpu240-summary.json @@ -0,0 +1,82 @@ +[ + { + "model": "qwen06b", + "native": { + "passed": true, + "concurrent_exact": 12, + "concurrent_total": 12, + "remote_prefills": 0 + }, + "pd": { + "passed": true, + "concurrent_exact": 8, + "concurrent_total": 12, + "remote_prefills": 21 + }, + "native_pd_serial_equal": { + "128": true, + "2048": true, + "8192": true + } + }, + { + "model": "qwen4b", + "native": { + "passed": true, + "concurrent_exact": 12, + "concurrent_total": 12, + "remote_prefills": 0 + }, + "pd": { + "passed": true, + "concurrent_exact": 12, + "concurrent_total": 12, + "remote_prefills": 21 + }, + "native_pd_serial_equal": { + "128": true, + "2048": true, + "8192": true + } + }, + { + "model": "koni", + "native": { + "passed": true, + "concurrent_exact": 12, + "concurrent_total": 12, + "remote_prefills": 0 + }, + "pd": { + "passed": true, + "concurrent_exact": 7, + "concurrent_total": 12, + "remote_prefills": 21 + }, + "native_pd_serial_equal": { + "128": true, + "2048": true, + "8192": true + } + }, + { + "model": "gemma", + "native": { + "passed": true, + "concurrent_exact": 12, + "concurrent_total": 12, + "remote_prefills": 0 + }, + "pd": { + "passed": true, + "concurrent_exact": 8, + "concurrent_total": 12, + "remote_prefills": 21 + }, + "native_pd_serial_equal": { + "128": true, + "2048": false, + "8192": true + } + } +] diff --git a/opt_prime/examples/pd/http_matrix/experiments/2026-10-23-gpu-live-pd240.md b/opt_prime/examples/pd/http_matrix/experiments/2026-10-23-gpu-live-pd240.md new file mode 100644 index 00000000..fd3b93af --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/experiments/2026-10-23-gpu-live-pd240.md @@ -0,0 +1,51 @@ +# Four-model live GPU PD HTTP validation + +- **Date**: 2026-10-23 +- **Task**: aicomp #240 +- **Host / device**: GPU host, two A100 80GB PCIe; Torch2.13.0+cu130, Transformers5.15.0 +- **Commit**: core369c8ff plus exp/240-gpu-live-pd validator edits; HTTP9e442cd loaded process-locally + +## Question + +Do real GPU-prefill/GPU-decode HTTP requests work for all four models across128/2048/8192tokens, queued concurrency4/8 and lifecycle events? Are concurrent outputs identical to each configuration's serial baseline? + +## Setup + +Existing HTTP validator now forwards decode and prefill device selection to serve_batched.build and requires actual remote prefill/transfer counters. Decode uses GPU1 with4slots, context9216, page64, local chunk256; optional producer uses GPU0 and existing TorchPrefillEngine defaults (chunk2048 and producer prefix cache enabled). Decoder prefix cache is initially disabled. Output length32 ignoring EOS; stream cancellation uses256tokens. This is a lifecycle/numerical matrix, not a semantic quality or matched speedup experiment. Producer cache policy differs from decoder cache policy and is reported explicitly. + +Each process initializes its own model(s). Existing SIGPIPE-safe HTTP9e442cd is injected by a local runner before loading validate.py. Other workloads are not touched. GPU standalone and PD use the same patched HTTP module. + +## Command + +From the aicomp worktree with its package on PYTHONPATH: + +```bash +git show 9e442cd:opt_prime/opt_prime/accel/serve/llmd.py > /tmp/pd-http-safe.py +CUDA_VISIBLE_DEVICES=0,1 PYTHONPATH=opt_prime python opt_prime/examples/pd/http_matrix/validate.py \ + --http-module /tmp/pd-http-safe.py --model "$MODEL" --output "$OUTPUT" \ + --device cuda:1 --pd-prefill-device cuda:0 +# Native control: omit --pd-prefill-device. Precision control: add --dtype float32. +# Public reproducibility controls (factory fa753e1, #241/#242): +# add --pd-prefill-no-prefix-cache --pd-prefill-chunk 256 +``` + +Use each selected locally cached model as MODEL. Decoder defaults remain4slots/context9216/page64/local chunk256; outputs32ignoringEOS. Original exact local queues/runners are /tmp/run_gpu240_queue.sh,/tmp/run_gpu240_controls.sh,/tmp/run_gpu240_more_controls.sh,/tmp/run_gpu240_chunk.sh and /tmp/run_gpu241_queue.sh. Earlier constructor controls used PREFILL_CACHE_OFF=1 and optionally PREFILL_CHUNK=256 in /tmp/run_gpu240_control.py; these are explicitly separated from later public-flag results. Raw per-model JSON/log are in analysis/serving-matrix-201. Selected shareable JSON accompanies this record in evidence240 and evidence241. + +## Results + +- Before validator wiring, the requested device flags fail argparse (unrecognized arguments). +- Qwen3-0.6B BF16 PD: lifecycle passes;21remote prefills,44544transferred tokens,0decode cached tokens,35072producer cached tokens. Concurrent equality8/12; both cancellation/disconnect reuse checks pass, invalid input returns400, no remaining request. This is not a numerical parity pass. +- All8 BF16 native/PD processes completed successfully. Each PD model reports21remote prefills (84total). All8 lifecycle/cancel/disconnect/invalid-input checks pass. Native concurrent equality48/48; PD concurrent equality35/48 (Qwen0.6B8/12,Qwen4B12/12,KONI7/12,Gemma8/12). +- Native-vs-PD serial32-token equality11/12 across model/length pairs; Gemma2048 differs. Qwen0.6B concurrent mismatches occur at2048tokens, from the second output token. FP32 and producer-cache-off controls complete below. +- FP32 native and PD for Qwen0.6B,KONI,Gemma: all6 lifecycle runs pass; each route36/36concurrent exact, native/PD9/9serial exact. Thus all originally divergent BF16 cases recover in FP32. +- BF16 producer-cache-off controls: all3 models12/12concurrent exact (36/36), but serial native/PD equality is7/9: Qwen0.6B2K and Gemma2K still differ. Cache-off alone is not full native parity. +- BF16 cache-off plus producer chunk256 (matching the native chunk) for Qwen0.6B and Gemma: both12/12concurrent and3/3native/PD serial exact. These are process-local construction controls, no transport change. Different cache/chunk execution shapes explain the observed tested differences more directly than transfer corruption; this is not a proof for arbitrary inputs. +- Public producer cache/chunk controls are implemented/tested under#241. Four-model public-flag validation completed:48/48concurrent equality,12/12native-PD serial equality,84remote prefills,0producer cache hits. All lifecycle checks pass (gpu241-summary.json). + +## Conclusion + +Real GPU PD is exercised, distinct from stored-KV replay. The observed Qwen BF16 difference must remain visible and be checked against native and FP32 controls. No speedup or producer-compute/copy overlap claim. Cross-host live RNGD-to-GPU remains separate work. + +## Validator integration check + +The first self-contained --http-module run failed: loading the HTTP candidate under a second module name created a distinct StageError class from the owner's class, so an expected400 became500. That failed JSON is retained as gpu240-cli-qwen06b-http-module-failed.json. The loader now imports the candidate under its canonical module name before consumers. The exact self-contained command then passed Qwen3-0.6B:12/12concurrent equality,21remote prefills,0producer cache hits,chunk256 and all lifecycle checks. This was a validator import-order defect, not evidence of a model or transfer failure. The earlier four-model evidence used canonical preload and is unaffected. diff --git a/opt_prime/examples/pd/http_matrix/validate.py b/opt_prime/examples/pd/http_matrix/validate.py index 7c907f46..ab19bcc4 100644 --- a/opt_prime/examples/pd/http_matrix/validate.py +++ b/opt_prime/examples/pd/http_matrix/validate.py @@ -1,6 +1,7 @@ """Actual HTTP validation through serve_batched.build; one model/configuration per process.""" import argparse +import importlib.util import itertools import json import sys @@ -16,20 +17,31 @@ from transformers import AutoTokenizer sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from opt_prime.accel.serve.llmd import serve_stage -from serve_batched import build def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument("--model", required=True) p.add_argument("--output", required=True) + p.add_argument("--http-module", help="trusted local HTTP module candidate for isolated validation") + p.add_argument("--device", default="cuda:0") + p.add_argument("--pd-prefill-device", default=None) + p.add_argument("--pd-prefill-no-prefix-cache", action="store_true") + p.add_argument("--pd-prefill-chunk", type=int, default=2048) p.add_argument("--prefix-cache", action="store_true") p.add_argument("--backend", choices=["torch", "atom"], default="torch") p.add_argument("--atom-kv-blocks", type=int, default=3) p.add_argument("--host-kv-gib", type=float, default=0) p.add_argument("--dtype", choices=["bfloat16", "float32"], default="bfloat16") a = p.parse_args() + if a.http_module: + spec = importlib.util.spec_from_file_location("opt_prime.accel.serve.llmd", a.http_module) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + # Load consumers after the candidate so StageError identity is shared across the server and owner. + from opt_prime.accel.serve.llmd import serve_stage + from serve_batched import build if a.backend == "atom" and a.dtype != "bfloat16": p.error("ATOM precision is determined by the compiled artifact") if a.host_kv_gib < 0 or (a.host_kv_gib and not a.prefix_cache): @@ -43,7 +55,7 @@ def main(): args = SimpleNamespace( model=a.model, backend=a.backend, - device="cuda:0", + device=a.device, dtype=a.dtype, slots=4 if a.backend == "torch" else 1, devices=devices, @@ -60,7 +72,9 @@ def main(): reload_bandwidth=None, prefill_rate=None, remote_kv=None, - pd_prefill_device=None, + pd_prefill_device=a.pd_prefill_device, + pd_prefill_no_prefix_cache=a.pd_prefill_no_prefix_cache, + pd_prefill_chunk=a.pd_prefill_chunk, pd_prefill_python=None, ) result = { @@ -72,9 +86,12 @@ def main(): "torch": torch.__version__, "dtype": a.dtype if a.backend == "torch" else "compiled artifact", "backend": a.backend, + "decode_device": a.device, + "pd_prefill_device": a.pd_prefill_device, + "producer_prefix_cache": None, "slots": args.slots, "atom_kv_blocks": a.atom_kv_blocks if a.backend == "atom" else None, - "device": torch.cuda.get_device_name(0) + "device": torch.cuda.get_device_name(torch.device(a.device)) if a.backend == "torch" else "allocated ATOM", "cases": [], @@ -83,6 +100,10 @@ def main(): output.parent.mkdir(parents=True, exist_ok=True) try: owner = build(args) + if a.pd_prefill_device: + producer = owner.service.batcher.worker.engine + result["producer_prefix_cache"] = producer.kv.prefix_cache + result["producer_chunk"] = producer.chunk except Exception as exc: result["startup_error"] = repr(exc) output.write_text(json.dumps(result, indent=2) + "\n") @@ -241,6 +262,11 @@ def client(): assert host_hits and result["host_hit_tokens"] > 0, ( "host tier not exercised" ) + result["batcher_stats"] = dict(owner.service.batcher.stats) + if a.pd_prefill_device: + # Nineteen completed normal requests; cancellations can add two more prefills. + assert result["batcher_stats"]["prefills"] >= 19 + assert result["batcher_stats"]["transferred_tokens"] > 0 result["passed"] = True except Exception as exc: # noqa: BLE001 - preserve failed hardware evidence result["error"] = repr(exc) @@ -260,6 +286,9 @@ def client(): server.shutdown() server.server_close() server_thread.join(5) + worker = getattr(owner.service.batcher, "worker", None) + if worker is not None: + worker.close() output.write_text(json.dumps(result, indent=2) + "\n") if not result["passed"]: raise RuntimeError(