diff --git a/opt_prime/examples/pd/http_matrix/.research/events.jsonl b/opt_prime/examples/pd/http_matrix/.research/events.jsonl new file mode 100644 index 00000000..c7f9dd30 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/.research/events.jsonl @@ -0,0 +1 @@ +{"id": "b61f99ae005a4258aa7f44d8edeca28d", "ts": "2026-10-23T18:08:44+09:00", "task": "Four-model HTTP matrix (aicomp #211)", "result": "All4 GPU models pass HTTP concurrency4/8, cancel/disconnect/drain/reuse checks. No-cacheBF16 and warm-cacheFP32 each48/48 concurrent outputs exact. Warm-cacheBF16 has31/48 raw equality; actualQwenHTTP tie explains observed branch. Warm-state cancellation isolates KONI cold-cache difference.", "next": "Publish GPUHTTP evidence; continue ATOM HTTP and RNGD routes. Preserve numerical/quality and true-overlap limitations.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"models": "4", "slots": "4", "concurrency": "4,8", "conditions": "bf16_no_cache,bf16_cache,fp32_cache"}, "artifacts": ["evidence/summary.json"], "experiments": ["experiments/2026-10-23-http-matrix211.md"], "labels": ["benchmark"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "GPU host", "branch": "main", "commit": "351d39d-dirty"} diff --git a/opt_prime/examples/pd/http_matrix/README.md b/opt_prime/examples/pd/http_matrix/README.md new file mode 100644 index 00000000..fa515f7c --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/README.md @@ -0,0 +1,11 @@ +# Four-model GPU HTTP validation (#211/#212) + +Run `validate.py --model MODEL --output result.json` under the engine environment with a free CUDA device. `--prefix-cache` enables the separate cache case; `--dtype float32` provides the numerical control. Offline immutable model snapshots are required. The default configuration is four paged execution slots, context9216, page64, prefill chunk256, inputs128/2048/8192 and32 fixed output tokens. This exercises the real HTTP service factory and sockets, not a mocked handler. + +`passed` covers HTTP lifecycle and completion counts. It does **not** assert equality of concurrent versus serial output: that is stored per response as `baseline_equal`. Cancellation compares warm-state output immediately before/after, and records cold-baseline equality separately. Cache-enabled runs require positive observed cache hits. Metrics are sampled after drain. + +`trace_prefix_http.py` uses the same CLI and records actual2K prefill boundaries and initial logits. `diagnose_prefix.py --model MODEL --output result.json` teacher-forces histories in the direct batcher. The direct probe did not reproduce the HTTP argmax change, so it must not replace actual HTTP evidence. An additional uncommitted allocation-mode probe changed only inference_mode to no_grad; its result is retained as a negative diagnostic in evidence. + +See experiments/2026-10-23-http-matrix211.md and evidence/summary.json. Evidence includes the initial failed KONI cold-baseline cancellation assertion. Earlier runs preceded the warm-state cancellation control and some metadata additions; their commands/configurations are recorded in the experiment. Public copies normalize local model-directory labels in metrics only; raw measurements remain in the research workspace. + +This GPU experiment does not complete ATOM/RNGD HTTP or PD coverage, long-context RNGD support, hybrid NPU cache restoration, broader semantic quality or true compute/transfer overlap. diff --git a/opt_prime/examples/pd/http_matrix/STATUS.md b/opt_prime/examples/pd/http_matrix/STATUS.md new file mode 100644 index 00000000..ed94583b --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/STATUS.md @@ -0,0 +1,3 @@ +# HTTP model matrix + +State: review; goal remains active. Four GPU models pass no-cache lifecycle checks and48/48 concurrent equality; BF16 warm cache has raw numerical differences. FP32 warm cache passes lifecycle and48/48 concurrent equality. Actual Qwen HTTP trace identifies a BF16 logit tie. Next: ATOM HTTP configurations and missing RNGD routes; broader semantic inputs and cache/host-tier ownership checks. diff --git a/opt_prime/examples/pd/http_matrix/diagnose_prefix.py b/opt_prime/examples/pd/http_matrix/diagnose_prefix.py new file mode 100644 index 00000000..b3bc7e2b --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/diagnose_prefix.py @@ -0,0 +1,143 @@ +"""Teacher-force equal histories while varying prefix-cache history and precision (#212).""" + +import argparse +import json +from pathlib import Path + +import torch +from opt_prime.accel.pd import PDRequest, SamplingParams, batching +from transformers import AutoModelForCausalLM, AutoTokenizer + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument("--model", required=True) +p.add_argument("--output", required=True) +a = p.parse_args() +torch.set_num_threads(4) +torch.backends.cuda.matmul.allow_tf32 = False +tok = AutoTokenizer.from_pretrained(a.model, local_files_only=True) +head = [tok.bos_token_id] if tok.bos_token_id is not None else [] +fill = tok.encode( + "This is background context. The red box contains apples. The blue box contains books.\n", + add_special_tokens=False, +) +tail = tok.encode( + "\nQuestion: What is the capital of France?\nAnswer:", add_special_tokens=False +) +inputs = { + n: head + + (fill * ((n - len(head) - len(tail) + len(fill) - 1) // len(fill)))[ + : n - len(head) - len(tail) + ] + + tail + for n in (128, 2048, 8192) +} +model = ( + AutoModelForCausalLM.from_pretrained( + a.model, local_files_only=True, dtype=torch.bfloat16 + ) + .cuda() + .eval() +) +results = [] +original = batching.select_token +try: + with torch.inference_mode(): + for dtype in ("bfloat16", "float32"): + model.to(getattr(torch, dtype)) + oracle = None + reference = None + for mode, history in [ + ("cold", []), + ("partial", [128]), + ("warm", [128, 2048, 8192]), + ("warm_concurrent", [128, 2048, 8192]), + ]: + b = batching.ContinuousBatcher( + model, + slots=4, + max_context=9216, + kv="paged", + page_size=64, + prefill_chunk=256, + prefix_cache=True, + ) + for i, n in enumerate(history): + f = b.submit(PDRequest(f"prime-{i}", tuple(inputs[n]), 32)) + b.run_until_idle() + f.result() + vectors = [] + predicted = [] + + def sample( + logits, + params, + position, + vectors=vectors, + predicted=predicted, + oracle=oracle, + ): + if params.seed != 212: + return original(logits, params, position) + vectors.append(logits.float().cpu().clone()) + predicted.append(int(logits.argmax())) + return ( + original(logits, params, position) + if oracle is None + else oracle[len(vectors) - 1] + ) + + batching.select_token = sample + before = b.kv.hit_tokens + f = b.submit( + PDRequest( + "target", tuple(inputs[2048]), 32, SamplingParams(seed=212) + ) + ) + if mode == "warm_concurrent": + for j, n in enumerate([128, 8192, 128]): + b.submit(PDRequest(f"companion-{j}", tuple(inputs[n]), 32)) + b.run_until_idle() + tokens = f.result() + batching.select_token = original + values = torch.stack(vectors) + assert values.shape[0] == 32 and torch.isfinite(values).all() + if oracle is None: + oracle = tokens + reference = values + delta = values - reference + different = [ + i for i, (u, v) in enumerate(zip(predicted, oracle)) if u != v + ] + rows = [] + for i in different: + tops = [z[i].topk(3) for z in (reference, values)] + rows.append( + { + "position": i, + "top_ids": [x.indices.tolist() for x in tops], + "top_logits": [x.values.tolist() for x in tops], + } + ) + result = { + "dtype": dtype, + "oracle": oracle, + "mode": mode, + "cache_hit_tokens": b.kv.hit_tokens - before, + "argmax_disagreements": different, + "max_abs_logit_difference": float(delta.abs().max()), + "relative_rms": float( + delta.square().mean().sqrt() / reference.square().mean().sqrt() + ), + "divergence_details": rows, + } + results.append(result) + print( + json.dumps( + {k: v for k, v in result.items() if k != "divergence_details"} + ), + flush=True, + ) + del b +finally: + batching.select_token = original +Path(a.output).write_text(json.dumps(results, indent=2) + "\n") diff --git a/opt_prime/examples/pd/http_matrix/evidence/gemma3-1b-cache-fp32.json b/opt_prime/examples/pd/http_matrix/evidence/gemma3-1b-cache-fp32.json new file mode 100644 index 00000000..db40d30f --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/gemma3-1b-cache-fp32.json @@ -0,0 +1,757 @@ +{ + "model_name": "Gemma3-1B", + "prefix_cache": true, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.10.0+cu128", + "dtype": "float32", + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 2.241513759974623, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 2.0816145570133813, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 160068, + 531, + 506, + 2934, + 236764, + 1144, + 563, + 506 + ], + "e2e_s": 2.145522948994767, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 2.208951373002492, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 2.2376674190163612, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 4.413927393004997, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 2.286759181006346, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 160068, + 531, + 506, + 2934, + 236764, + 1144, + 563, + 506 + ], + "e2e_s": 4.220815120992484, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 2.107757734018378, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 2.4119486029958352, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 160068, + 531, + 506, + 2934, + 236764, + 1144, + 563, + 506 + ], + "e2e_s": 2.1996703439799603, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 4.346329399995739, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 4.280958282994106, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 160068, + 531, + 506, + 2934, + 236764, + 1144, + 563, + 506 + ], + "e2e_s": 4.408146672998555, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 9.970391887996811 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 160068, + 531, + 506, + 2934, + 236764, + 1144, + 563, + 506 + ], + "e2e_s": 2.304752541007474 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 108, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 3.5468254250008613 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "after_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "after_tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Gemma3-1B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Gemma3-1B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Gemma3-1B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Gemma3-1B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Gemma3-1B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.04\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.06\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.25\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.75\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Gemma3-1B\"} 12.686048940988258\nvllm:time_to_first_token_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.025\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.05\"} 4\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.075\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.15\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.2\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.3\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Gemma3-1B\"} 1.3443234236479078\nvllm:time_per_output_token_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.8\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.5\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.0\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 13\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"5.0\"} 18\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Gemma3-1B\"} 54.36007507407339\nvllm:e2e_request_latency_seconds_count{model_name=\"Gemma3-1B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Gemma3-1B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Gemma3-1B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"True\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Gemma3-1B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Gemma3-1B\"} 35072\n", + "prefix_hit_tokens": 35072.0 +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/gemma3-1b-cache.json b/opt_prime/examples/pd/http_matrix/evidence/gemma3-1b-cache.json new file mode 100644 index 00000000..e806861f --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/gemma3-1b-cache.json @@ -0,0 +1,618 @@ +{ + "model_name": "Gemma3-1B", + "prefix_cache": true, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.10.0+cu128", + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 0.9464839909924194, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 0.8691790089942515, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 115943, + 236890, + 106, + 237021, + 3110, + 21647, + 236830, + 26920 + ], + "e2e_s": 0.9193995460227598, + "baseline_equal": false + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 0.8933476890088059, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 0.9437579740188085, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 1.8549946299754083, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 0.8839836969855241, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 115943, + 236890, + 106, + 237021, + 3110, + 21647, + 236830, + 26920 + ], + "e2e_s": 0.9343937560042832, + "baseline_equal": false + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 1.0447216070024297, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 0.9916092189960182, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 115943, + 236890, + 106, + 237021, + 3110, + 21647, + 236830, + 26920 + ], + "e2e_s": 1.7720146540086716, + "baseline_equal": false + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 1.7968444760190323, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 1.8223218229832128, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 115943, + 236890, + 106, + 237021, + 3110, + 21647, + 236830, + 26920 + ], + "e2e_s": 1.8476780339842662, + "baseline_equal": false + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 2.440668638009811 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 160068, + 531, + 506, + 2934, + 236764, + 1144, + 563, + 506 + ], + "e2e_s": 1.0322104019869585 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 1.5857249469845556 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Gemma3-1B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Gemma3-1B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Gemma3-1B\",finished_reason=\"length\"} 17\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Gemma3-1B\"} 44032\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Gemma3-1B\"} 544\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.04\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.06\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.08\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.25\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 12\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.75\"} 12\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 16\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"5.0\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"7.5\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"10.0\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 17\nvllm:time_to_first_token_seconds_sum{model_name=\"Gemma3-1B\"} 6.069310090941144\nvllm:time_to_first_token_seconds_count{model_name=\"Gemma3-1B\"} 17\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.025\"} 3\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.05\"} 16\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.075\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.15\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.2\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.3\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.4\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.75\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 17\nvllm:time_per_output_token_seconds_sum{model_name=\"Gemma3-1B\"} 0.48834834064473404\nvllm:time_per_output_token_seconds_count{model_name=\"Gemma3-1B\"} 17\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.8\"} 2\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 9\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.5\"} 11\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.0\"} 16\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"5.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"10.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"15.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"20.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"30.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"40.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"50.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"60.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 17\nvllm:e2e_request_latency_seconds_sum{model_name=\"Gemma3-1B\"} 21.208108650927898\nvllm:e2e_request_latency_seconds_count{model_name=\"Gemma3-1B\"} 17\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Gemma3-1B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Gemma3-1B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"True\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Gemma3-1B\"} 44288\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Gemma3-1B\"} 34944\n", + "prefix_hit_tokens": 34944.0 +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/gemma3-1b-no-cache.json b/opt_prime/examples/pd/http_matrix/evidence/gemma3-1b-no-cache.json new file mode 100644 index 00000000..5e849a44 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/gemma3-1b-no-cache.json @@ -0,0 +1,614 @@ +{ + "model_name": "Gemma3-1B", + "prefix_cache": false, + "passed": true, + "cases": [ + { + "concurrency": 4, + "wall_s": 2.8574796180182602, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 1.523203579010442, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 236743, + 107, + 106, + 107, + 106, + 107, + 106, + 107 + ], + "e2e_s": 1.9826781659794506, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 2.829485169990221, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 2.8537060209782794, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 5.4193066009902395, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 1.528549683978781, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 236743, + 107, + 106, + 107, + 106, + 107, + 106, + 107 + ], + "e2e_s": 2.0056576849892735, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 3.585067792009795, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 3.6416808770154603, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 236743, + 107, + 106, + 107, + 106, + 107, + 106, + 107 + ], + "e2e_s": 4.11344148000353, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 5.185657046997221, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 5.209825452999212, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 236743, + 107, + 106, + 107, + 106, + 107, + 106, + 107 + ], + "e2e_s": 5.411450558982324, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9079, + 107, + 106, + 107, + 106, + 130614, + 107, + 106, + 107, + 106, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695 + ], + "e2e_s": 4.973915248003323 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9079, + 107, + 106, + 106, + 237021, + 3110, + 21647, + 236830, + 26920, + 10986, + 5470, + 236764, + 16404, + 11729, + 12241, + 9927, + 45278, + 48153, + 12858, + 92449, + 6754, + 19585, + 236890, + 106, + 236743, + 107, + 106, + 107, + 106, + 107, + 106, + 107 + ], + "e2e_s": 0.9523018979816698 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 9079, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761, + 669, + 3730, + 3673, + 6097, + 6249, + 236761, + 107, + 2094, + 563, + 1695, + 4403, + 236761, + 669, + 2604, + 3673, + 6097, + 36157, + 236761 + ], + "e2e_s": 1.640141040988965 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Gemma3-1B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Gemma3-1B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Gemma3-1B\",finished_reason=\"length\"} 17\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Gemma3-1B\"} 44032\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Gemma3-1B\"} 544\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.04\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.06\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.08\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.25\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.75\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 9\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"5.0\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"7.5\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"10.0\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 17\nvllm:time_to_first_token_seconds_sum{model_name=\"Gemma3-1B\"} 26.016970497032162\nvllm:time_to_first_token_seconds_count{model_name=\"Gemma3-1B\"} 17\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.025\"} 3\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.05\"} 13\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.075\"} 16\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 16\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.15\"} 16\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.2\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.3\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.4\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.75\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 17\nvllm:time_per_output_token_seconds_sum{model_name=\"Gemma3-1B\"} 0.7374686076119917\nvllm:time_per_output_token_seconds_count{model_name=\"Gemma3-1B\"} 17\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.8\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 3\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.5\"} 3\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.0\"} 8\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"2.5\"} 8\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"5.0\"} 14\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"10.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"15.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"20.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"30.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"40.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"50.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"60.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 17\nvllm:e2e_request_latency_seconds_sum{model_name=\"Gemma3-1B\"} 48.87849733300391\nvllm:e2e_request_latency_seconds_count{model_name=\"Gemma3-1B\"} 17\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Gemma3-1B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Gemma3-1B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Gemma3-1B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Gemma3-1B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Gemma3-1B\"} 44288\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Gemma3-1B\"} 0\n" +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/koni-8b-cache-fp32.json b/opt_prime/examples/pd/http_matrix/evidence/koni-8b-cache-fp32.json new file mode 100644 index 00000000..f55e6498 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/koni-8b-cache-fp32.json @@ -0,0 +1,757 @@ +{ + "model_name": "KONI-8B", + "prefix_cache": true, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.10.0+cu128", + "dtype": "float32", + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 2.8278144180076197, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 2.6412628809921443, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 2.715809443994658, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 2.7903507009905297, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 2.8252312569820788, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 5.551692591019673, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 2.8347582390124444, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 2.6620676720049232, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 3.0513848479895387, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 3.209095676022116, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 5.374735250021331, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 5.448900168004911, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 5.495637938991422, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 5.541837418015348, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 2.7471833800082095 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 3.453492267988622 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 9.655912321002688 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "after_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "after_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"KONI-8B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"KONI-8B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"KONI-8B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"KONI-8B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"KONI-8B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"KONI-8B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"KONI-8B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"KONI-8B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.04\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.06\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.08\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.25\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.75\"} 13\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 13\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"5.0\"} 18\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"KONI-8B\"} 24.675631930993404\nvllm:time_to_first_token_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.025\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.05\"} 5\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.075\"} 8\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"KONI-8B\"} 1.302136850934814\nvllm:time_per_output_token_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.8\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"1.5\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"2.0\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"5.0\"} 14\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"KONI-8B\"} 65.04187430997263\nvllm:e2e_request_latency_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"KONI-8B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"KONI-8B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"KONI-8B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"True\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"KONI-8B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"KONI-8B\"} 35072\n", + "prefix_hit_tokens": 35072.0 +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/koni-8b-cache-warm-cancel.json b/opt_prime/examples/pd/http_matrix/evidence/koni-8b-cache-warm-cancel.json new file mode 100644 index 00000000..be500adc --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/koni-8b-cache-warm-cancel.json @@ -0,0 +1,756 @@ +{ + "model_name": "KONI-8B", + "prefix_cache": true, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.10.0+cu128", + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 1.151520660991082, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 1.0571194030053448, + "baseline_equal": false + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 1.0876679999928456, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 1.1503521429840475, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 1.1194605989730917, + "baseline_equal": false + } + ] + }, + { + "concurrency": 8, + "wall_s": 2.2306854949856643, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 1.0713792240130715, + "baseline_equal": false + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 1.224772916000802, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 1.134158019995084, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 1.2769538759894203, + "baseline_equal": false + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 2.151573367009405, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 2.181768304988509, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 2.203191770997364, + "baseline_equal": false + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 2.2257905150181614, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 4495, + 30, + 7566, + 627, + 15546, + 374, + 279, + 4900, + 315, + 9822, + 5380, + 16533, + 25, + 578, + 4900, + 315, + 9822, + 374, + 61544, + 63401, + 627 + ], + "e2e_s": 2.2818681340140756 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 1.0470782949996646 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 2.550022681010887 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": false, + "before_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "after_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": false, + "before_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "after_tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"KONI-8B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"KONI-8B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"KONI-8B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"KONI-8B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"KONI-8B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"KONI-8B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"KONI-8B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"KONI-8B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.04\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 7\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.25\"} 12\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.75\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"KONI-8B\"} 8.086359513981733\nvllm:time_to_first_token_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.025\"} 5\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.05\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"KONI-8B\"} 0.5792460842931177\nvllm:time_per_output_token_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.8\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"1.5\"} 13\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"2.0\"} 13\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 18\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"5.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"KONI-8B\"} 26.042988127068384\nvllm:e2e_request_latency_seconds_count{model_name=\"KONI-8B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"KONI-8B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"KONI-8B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"KONI-8B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"True\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"KONI-8B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"KONI-8B\"} 35072\n", + "prefix_hit_tokens": 35072.0 +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/koni-8b-cache.json b/opt_prime/examples/pd/http_matrix/evidence/koni-8b-cache.json new file mode 100644 index 00000000..12eeb871 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/koni-8b-cache.json @@ -0,0 +1,607 @@ +{ + "model_name": "KONI-8B", + "prefix_cache": true, + "passed": false, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.10.0+cu128", + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 1.1467661240021698, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 1.0544188659987412, + "baseline_equal": false + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 1.0847634740057401, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 1.145477481011767, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 1.114083694992587, + "baseline_equal": false + } + ] + }, + { + "concurrency": 8, + "wall_s": 2.206508102011867, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 1.0507869120046962, + "baseline_equal": false + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 1.1138840070052538, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 1.2566481890098657, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 1.202803769003367, + "baseline_equal": false + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 2.127927376976004, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 2.156963574991096, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 11190, + 30, + 1442, + 10035, + 11, + 4587, + 1095, + 757, + 1440, + 422, + 499, + 1205, + 904, + 4726, + 13291, + 13, + 720, + 2746, + 539, + 11, + 4587 + ], + "e2e_s": 2.178787862008903, + "baseline_equal": false + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 2.1991784280107822, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 4495, + 30, + 7566, + 627, + 15546, + 374, + 279, + 4900, + 315, + 9822, + 5380, + 16533, + 25, + 578, + 4900, + 315, + 9822, + 374, + 61544, + 63401, + 627 + ], + "e2e_s": 2.1833766409836244 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 6366, + 18112, + 11, + 1989, + 51677 + ], + "e2e_s": 1.0511275099997874 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 2.539744970999891 + } + }, + "error": "AssertionError('post-cancellation output changed')" +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/koni-8b-no-cache.json b/opt_prime/examples/pd/http_matrix/evidence/koni-8b-no-cache.json new file mode 100644 index 00000000..4f49545c --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/koni-8b-no-cache.json @@ -0,0 +1,614 @@ +{ + "model_name": "KONI-8B", + "prefix_cache": false, + "passed": true, + "cases": [ + { + "concurrency": 4, + "wall_s": 4.067219550022855, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 4495, + 30, + 7566, + 627, + 15546, + 374, + 279, + 4900, + 315, + 9822, + 5380, + 16533, + 25, + 578, + 4900, + 315, + 9822, + 374, + 61544, + 63401, + 627 + ], + "e2e_s": 2.1317082040186506, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 2.87993640001514, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 4.065516554983333, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 4495, + 30, + 7566, + 627, + 15546, + 374, + 279, + 4900, + 315, + 9822, + 5380, + 16533, + 25, + 578, + 4900, + 315, + 9822, + 374, + 61544, + 63401, + 627 + ], + "e2e_s": 2.9753001790086273, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 7.984032834996469, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 4495, + 30, + 7566, + 627, + 15546, + 374, + 279, + 4900, + 315, + 9822, + 5380, + 16533, + 25, + 578, + 4900, + 315, + 9822, + 374, + 61544, + 63401, + 627 + ], + "e2e_s": 2.1807764480181504, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 2.9363406949851196, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 5.706791567004984, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 4495, + 30, + 7566, + 627, + 15546, + 374, + 279, + 4900, + 315, + 9822, + 5380, + 16533, + 25, + 578, + 4900, + 315, + 9822, + 374, + 61544, + 63401, + 627 + ], + "e2e_s": 5.802898477006238, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 7.781984970992198, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 7.5999276009970345, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 4495, + 30, + 7566, + 627, + 15546, + 374, + 279, + 4900, + 315, + 9822, + 5380, + 16533, + 25, + 578, + 4900, + 315, + 9822, + 374, + 61544, + 63401, + 627 + ], + "e2e_s": 7.802780869998969, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 7.974798587994883, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 720, + 3957, + 420, + 2038, + 4495, + 30, + 7566, + 627, + 15546, + 374, + 279, + 4900, + 315, + 9822, + 5380, + 16533, + 25, + 578, + 4900, + 315, + 9822, + 374, + 61544, + 63401, + 627 + ], + "e2e_s": 2.3734899570117705 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 12366, + 198, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 13, + 1102, + 374, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 323, + 374, + 3967, + 369, + 1202, + 9257, + 3925, + 11, + 13042, + 61024 + ], + "e2e_s": 1.0535996099933982 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12366, + 13, + 720, + 70869, + 25, + 12366, + 374, + 279, + 6864, + 3363, + 315, + 9822, + 11, + 7559, + 304, + 279, + 18671, + 961, + 315, + 279, + 3224, + 13, + 1102, + 374, + 3967, + 369, + 1202, + 27373, + 61024, + 1778, + 439, + 279 + ], + "e2e_s": 2.8337559860083275 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"KONI-8B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"KONI-8B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"KONI-8B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"KONI-8B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"KONI-8B\",finished_reason=\"length\"} 17\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"KONI-8B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"KONI-8B\"} 44032\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"KONI-8B\"} 544\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.04\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.06\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.08\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.25\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.75\"} 9\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 9\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"5.0\"} 13\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"7.5\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"10.0\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 17\nvllm:time_to_first_token_seconds_sum{model_name=\"KONI-8B\"} 39.7749398619635\nvllm:time_to_first_token_seconds_count{model_name=\"KONI-8B\"} 17\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.025\"} 3\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.05\"} 8\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.075\"} 12\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.15\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.2\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.3\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.4\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"0.75\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 17\nvllm:time_per_output_token_seconds_sum{model_name=\"KONI-8B\"} 0.8823879668395215\nvllm:time_per_output_token_seconds_count{model_name=\"KONI-8B\"} 17\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"0.8\"} 2\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 2\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"1.5\"} 3\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"2.0\"} 3\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"2.5\"} 6\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"5.0\"} 11\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"10.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"15.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"20.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"30.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"40.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"50.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"60.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 17\nvllm:e2e_request_latency_seconds_sum{model_name=\"KONI-8B\"} 67.12896683398867\nvllm:e2e_request_latency_seconds_count{model_name=\"KONI-8B\"} 17\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"KONI-8B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"KONI-8B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"KONI-8B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"KONI-8B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"KONI-8B\"} 44288\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"KONI-8B\"} 0\n" +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-cache-fp32.json b/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-cache-fp32.json new file mode 100644 index 00000000..d34bc7fb --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-cache-fp32.json @@ -0,0 +1,757 @@ +{ + "model_name": "Qwen3-0.6B", + "prefix_cache": true, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.10.0+cu128", + "dtype": "float32", + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 2.402447297994513, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.238832728995476, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 2.310036359995138, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.381743968988303, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.400144301005639, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 4.691692629014142, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.2527011430065613, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 2.342333090986358, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.4802317889989354, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 4.553602793021128, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 2.566094036010327, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 4.622277185990242, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 4.652243382006418, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 4.684206027974142, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.7310442920133937 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 1.2759598580014426 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 3.671703375002835 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Qwen3-0.6B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Qwen3-0.6B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Qwen3-0.6B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Qwen3-0.6B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Qwen3-0.6B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.02\"} 3\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.04\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.08\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.1\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.25\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.75\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.5\"} 18\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Qwen3-0.6B\"} 12.879286709940061\nvllm:time_to_first_token_seconds_count{model_name=\"Qwen3-0.6B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.025\"} 4\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.05\"} 5\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.075\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Qwen3-0.6B\"} 1.1069311353578908\nvllm:time_per_output_token_seconds_count{model_name=\"Qwen3-0.6B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.8\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.5\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.0\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.5\"} 12\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"5.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Qwen3-0.6B\"} 47.194151906034676\nvllm:e2e_request_latency_seconds_count{model_name=\"Qwen3-0.6B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Qwen3-0.6B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Qwen3-0.6B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"True\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Qwen3-0.6B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Qwen3-0.6B\"} 35072\n", + "prefix_hit_tokens": 35072.0 +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-cache-fp32.trace.json b/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-cache-fp32.trace.json new file mode 100644 index 00000000..5b9a35ca --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-cache-fp32.trace.json @@ -0,0 +1,250 @@ +{ + "samples": [ + { + "position": 2048, + "ids": [ + 9625, + 12095, + 576, + 220, + 362 + ], + "logits": [ + 17.970117568969727, + 17.2199764251709, + 16.322154998779297, + 15.921201705932617, + 15.71206283569336 + ] + }, + { + "position": 2049, + "ids": [ + 594, + 374, + 702, + 198, + 624 + ], + "logits": [ + 18.322538375854492, + 18.220041275024414, + 17.7713680267334, + 16.82594871520996, + 16.29151725769043 + ] + }, + { + "position": 2048, + "ids": [ + 9625, + 12095, + 576, + 220, + 362 + ], + "logits": [ + 17.970115661621094, + 17.219989776611328, + 16.32215118408203, + 15.921213150024414, + 15.712068557739258 + ] + }, + { + "position": 2049, + "ids": [ + 594, + 374, + 702, + 198, + 624 + ], + "logits": [ + 18.322528839111328, + 18.22003936767578, + 17.7713623046875, + 16.825960159301758, + 16.291519165039062 + ] + }, + { + "position": 2048, + "ids": [ + 9625, + 12095, + 576, + 220, + 362 + ], + "logits": [ + 17.970115661621094, + 17.219989776611328, + 16.32215118408203, + 15.921213150024414, + 15.712068557739258 + ] + }, + { + "position": 2049, + "ids": [ + 594, + 374, + 702, + 198, + 624 + ], + "logits": [ + 18.322528839111328, + 18.22003936767578, + 17.7713623046875, + 16.825960159301758, + 16.291519165039062 + ] + }, + { + "position": 2048, + "ids": [ + 9625, + 12095, + 576, + 220, + 362 + ], + "logits": [ + 17.970115661621094, + 17.219989776611328, + 16.32215118408203, + 15.921213150024414, + 15.712068557739258 + ] + }, + { + "position": 2049, + "ids": [ + 594, + 374, + 702, + 198, + 624 + ], + "logits": [ + 18.322528839111328, + 18.22003936767578, + 17.7713623046875, + 16.825960159301758, + 16.291519165039062 + ] + }, + { + "position": 2048, + "ids": [ + 9625, + 12095, + 576, + 220, + 362 + ], + "logits": [ + 17.970115661621094, + 17.219989776611328, + 16.32215118408203, + 15.921213150024414, + 15.712068557739258 + ] + }, + { + "position": 2049, + "ids": [ + 594, + 374, + 702, + 198, + 624 + ], + "logits": [ + 18.322528839111328, + 18.22003936767578, + 17.7713623046875, + 16.825960159301758, + 16.291519165039062 + ] + } + ], + "prefills": [ + { + "request": "http211-2", + "start": 64, + "end": 320, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-2", + "start": 320, + "end": 576, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-2", + "start": 576, + "end": 832, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-2", + "start": 832, + "end": 1088, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-2", + "start": 1088, + "end": 1344, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-2", + "start": 1344, + "end": 1600, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-2", + "start": 1600, + "end": 1856, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-2", + "start": 1856, + "end": 2048, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-5", + "start": 1984, + "end": 2048, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-9", + "start": 1984, + "end": 2048, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-12", + "start": 1984, + "end": 2048, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-15", + "start": 1984, + "end": 2048, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + } + ], + "attention": "sdpa", + "dtype": "torch.float32" +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-cache-traced.json b/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-cache-traced.json new file mode 100644 index 00000000..ae49ae86 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-cache-traced.json @@ -0,0 +1,756 @@ +{ + "model_name": "Qwen3-0.6B", + "prefix_cache": true, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.10.0+cu128", + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 0.8206465049879625, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.7536621520121116, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 0.774523401982151, + "baseline_equal": false + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.7975727159937378, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.8182071849878412, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 1.5687581850215793, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.7653392929933034, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 0.809300165012246, + "baseline_equal": false + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.872840943018673, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.5055830689962022, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 0.9116728929802775, + "baseline_equal": false + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.5256405679974705, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.5443249060190283, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 1.5597090859955642, + "baseline_equal": false + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.1282861449872144 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 0.8239581779926084 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.5181420689914376 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "after_tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Qwen3-0.6B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Qwen3-0.6B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Qwen3-0.6B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Qwen3-0.6B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Qwen3-0.6B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.02\"} 1\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.04\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.08\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.1\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.25\"} 13\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.75\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Qwen3-0.6B\"} 5.61329858697718\nvllm:time_to_first_token_seconds_count{model_name=\"Qwen3-0.6B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.025\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.05\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.075\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.1\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Qwen3-0.6B\"} 0.4468158139986166\nvllm:time_per_output_token_seconds_count{model_name=\"Qwen3-0.6B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.8\"} 8\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 13\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.5\"} 14\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.0\"} 18\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"2.5\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"5.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Qwen3-0.6B\"} 19.464588820934296\nvllm:e2e_request_latency_seconds_count{model_name=\"Qwen3-0.6B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-0.6B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Qwen3-0.6B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Qwen3-0.6B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Qwen3-0.6B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"True\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Qwen3-0.6B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Qwen3-0.6B\"} 35072\n", + "prefix_hit_tokens": 35072.0 +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-cache-traced.trace.json b/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-cache-traced.trace.json new file mode 100644 index 00000000..a3518bce --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-cache-traced.trace.json @@ -0,0 +1,250 @@ +{ + "samples": [ + { + "position": 2048, + "ids": [ + 9625, + 12095, + 576, + 220, + 362 + ], + "logits": [ + 17.75, + 17.125, + 16.125, + 15.875, + 15.625 + ] + }, + { + "position": 2049, + "ids": [ + 374, + 594, + 702, + 198, + 624 + ], + "logits": [ + 18.375, + 18.375, + 17.875, + 16.875, + 16.375 + ] + }, + { + "position": 2048, + "ids": [ + 9625, + 12095, + 576, + 220, + 362 + ], + "logits": [ + 17.75, + 17.25, + 16.125, + 15.8125, + 15.6875 + ] + }, + { + "position": 2049, + "ids": [ + 594, + 374, + 702, + 198, + 624 + ], + "logits": [ + 18.5, + 18.25, + 17.75, + 16.875, + 16.375 + ] + }, + { + "position": 2048, + "ids": [ + 9625, + 12095, + 576, + 220, + 362 + ], + "logits": [ + 17.75, + 17.25, + 16.125, + 15.8125, + 15.6875 + ] + }, + { + "position": 2049, + "ids": [ + 594, + 374, + 702, + 198, + 624 + ], + "logits": [ + 18.5, + 18.25, + 17.75, + 16.875, + 16.375 + ] + }, + { + "position": 2048, + "ids": [ + 9625, + 12095, + 576, + 220, + 362 + ], + "logits": [ + 17.75, + 17.25, + 16.125, + 15.8125, + 15.6875 + ] + }, + { + "position": 2049, + "ids": [ + 594, + 374, + 702, + 198, + 624 + ], + "logits": [ + 18.5, + 18.25, + 17.75, + 16.875, + 16.375 + ] + }, + { + "position": 2048, + "ids": [ + 9625, + 12095, + 576, + 220, + 362 + ], + "logits": [ + 17.75, + 17.25, + 16.125, + 15.8125, + 15.6875 + ] + }, + { + "position": 2049, + "ids": [ + 594, + 374, + 702, + 198, + 624 + ], + "logits": [ + 18.5, + 18.25, + 17.75, + 16.875, + 16.375 + ] + } + ], + "prefills": [ + { + "request": "http211-2", + "start": 64, + "end": 320, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-2", + "start": 320, + "end": 576, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-2", + "start": 576, + "end": 832, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-2", + "start": 832, + "end": 1088, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-2", + "start": 1088, + "end": 1344, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-2", + "start": 1344, + "end": 1600, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-2", + "start": 1600, + "end": 1856, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-2", + "start": 1856, + "end": 2048, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-5", + "start": 1984, + "end": 2048, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-9", + "start": 1984, + "end": 2048, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-12", + "start": 1984, + "end": 2048, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + }, + { + "request": "http211-15", + "start": 1984, + "end": 2048, + "input_sha256": "aaf867b52860318747d526952857333f802504db263680cceecdd31c2eb2581a" + } + ], + "attention": "sdpa", + "dtype": "torch.bfloat16" +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-cache.json b/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-cache.json new file mode 100644 index 00000000..49d36e74 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-cache.json @@ -0,0 +1,614 @@ +{ + "model_name": "Qwen3-0.6B", + "prefix_cache": true, + "passed": true, + "cases": [ + { + "concurrency": 4, + "wall_s": 0.8051764970005024, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.7389973659883253, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 0.8043028900283389, + "baseline_equal": false + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.7607127290102653, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.7812860289996024, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 1.5730774590047076, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.7571643099945504, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 0.8642189030069858, + "baseline_equal": false + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.799529132025782, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 0.9051935940224212, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 1.5074207639845554, + "baseline_equal": false + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.5267920799960848, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.5657571490155533, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 1.546000788017409, + "baseline_equal": false + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.142416154005332 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9625, + 374, + 264, + 3146, + 7407, + 304, + 4505, + 13, + 11445, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374 + ], + "e2e_s": 0.775701372011099 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.5233713439956773 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Qwen/Qwen3-0.6B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Qwen/Qwen3-0.6B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Qwen/Qwen3-0.6B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Qwen/Qwen3-0.6B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Qwen/Qwen3-0.6B\",finished_reason=\"length\"} 17\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Qwen/Qwen3-0.6B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Qwen/Qwen3-0.6B\"} 44032\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Qwen/Qwen3-0.6B\"} 544\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.02\"} 1\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.04\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.06\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.08\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.25\"} 11\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.5\"} 12\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.75\"} 13\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"1.0\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"2.5\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"5.0\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"7.5\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"10.0\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"+Inf\"} 17\nvllm:time_to_first_token_seconds_sum{model_name=\"Qwen/Qwen3-0.6B\"} 5.504803531977814\nvllm:time_to_first_token_seconds_count{model_name=\"Qwen/Qwen3-0.6B\"} 17\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.025\"} 16\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.05\"} 16\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.075\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.1\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.15\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.2\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.3\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.4\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.5\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.75\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"1.0\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"2.5\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"+Inf\"} 17\nvllm:time_per_output_token_seconds_sum{model_name=\"Qwen/Qwen3-0.6B\"} 0.4053757695477426\nvllm:time_per_output_token_seconds_count{model_name=\"Qwen/Qwen3-0.6B\"} 17\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.8\"} 9\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"1.0\"} 11\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"1.5\"} 12\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"2.0\"} 16\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"2.5\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"5.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"10.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"15.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"20.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"30.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"40.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"50.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"60.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"+Inf\"} 17\nvllm:e2e_request_latency_seconds_sum{model_name=\"Qwen/Qwen3-0.6B\"} 18.071452387957834\nvllm:e2e_request_latency_seconds_count{model_name=\"Qwen/Qwen3-0.6B\"} 17\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Qwen/Qwen3-0.6B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Qwen/Qwen3-0.6B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Qwen/Qwen3-0.6B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"True\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Qwen/Qwen3-0.6B\"} 44288\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Qwen/Qwen3-0.6B\"} 34944\n" +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-no-cache.json b/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-no-cache.json new file mode 100644 index 00000000..03ec1ced --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-no-cache.json @@ -0,0 +1,614 @@ +{ + "model_name": "Qwen3-0.6B", + "prefix_cache": false, + "passed": true, + "cases": [ + { + "concurrency": 4, + "wall_s": 2.5172985260142013, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.2786470860010013, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 1.7402843740128446, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.4957400189887267, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.5137767670094036, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 4.5096583620179445, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.5881094159849454, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 1.5286925920227077, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 2.9706211669836193, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 3.0231258110143244, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 3.467287645005854, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 4.341163416975178, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 4.360203488002298, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 4.500361037993571, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 9625, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 4.044220726005733 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "e2e_s": 0.742115142988041 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 12095, + 198, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303 + ], + "e2e_s": 1.593658700992819 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Qwen/Qwen3-0.6B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Qwen/Qwen3-0.6B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Qwen/Qwen3-0.6B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Qwen/Qwen3-0.6B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Qwen/Qwen3-0.6B\",finished_reason=\"length\"} 17\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Qwen/Qwen3-0.6B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Qwen/Qwen3-0.6B\"} 44032\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Qwen/Qwen3-0.6B\"} 544\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.02\"} 2\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.04\"} 3\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.06\"} 3\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.08\"} 3\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.1\"} 3\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.25\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.5\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.75\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"1.0\"} 9\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"2.5\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"5.0\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"7.5\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"10.0\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"+Inf\"} 17\nvllm:time_to_first_token_seconds_sum{model_name=\"Qwen/Qwen3-0.6B\"} 22.171081921929726\nvllm:time_to_first_token_seconds_count{model_name=\"Qwen/Qwen3-0.6B\"} 17\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.025\"} 8\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.05\"} 16\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.075\"} 16\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.1\"} 16\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.15\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.2\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.3\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.4\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.5\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.75\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"1.0\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"2.5\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"+Inf\"} 17\nvllm:time_per_output_token_seconds_sum{model_name=\"Qwen/Qwen3-0.6B\"} 0.6135181779391132\nvllm:time_per_output_token_seconds_count{model_name=\"Qwen/Qwen3-0.6B\"} 17\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.8\"} 3\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"1.0\"} 3\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"1.5\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"2.0\"} 8\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"2.5\"} 9\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"5.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"10.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"15.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"20.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"30.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"40.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"50.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"60.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"+Inf\"} 17\nvllm:e2e_request_latency_seconds_sum{model_name=\"Qwen/Qwen3-0.6B\"} 41.19014543804224\nvllm:e2e_request_latency_seconds_count{model_name=\"Qwen/Qwen3-0.6B\"} 17\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen/Qwen3-0.6B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Qwen/Qwen3-0.6B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Qwen/Qwen3-0.6B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Qwen/Qwen3-0.6B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Qwen/Qwen3-0.6B\"} 44288\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Qwen/Qwen3-0.6B\"} 0\n" +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-prefix-concurrent-numerics.json b/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-prefix-concurrent-numerics.json new file mode 100644 index 00000000..23497494 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-prefix-concurrent-numerics.json @@ -0,0 +1,346 @@ +[ + { + "dtype": "bfloat16", + "oracle": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "mode": "cold", + "cache_hit_tokens": 0, + "argmax_disagreements": [], + "max_abs_logit_difference": 0.0, + "relative_rms": 0.0, + "divergence_details": [] + }, + { + "dtype": "bfloat16", + "oracle": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "mode": "partial", + "cache_hit_tokens": 64, + "argmax_disagreements": [], + "max_abs_logit_difference": 0.5, + "relative_rms": 0.022843077778816223, + "divergence_details": [] + }, + { + "dtype": "bfloat16", + "oracle": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "mode": "warm", + "cache_hit_tokens": 1984, + "argmax_disagreements": [], + "max_abs_logit_difference": 0.515625, + "relative_rms": 0.023976627737283707, + "divergence_details": [] + }, + { + "dtype": "bfloat16", + "oracle": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "mode": "warm_concurrent", + "cache_hit_tokens": 10240, + "argmax_disagreements": [], + "max_abs_logit_difference": 0.515625, + "relative_rms": 0.023976627737283707, + "divergence_details": [] + }, + { + "dtype": "float32", + "oracle": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "mode": "cold", + "cache_hit_tokens": 0, + "argmax_disagreements": [], + "max_abs_logit_difference": 0.0, + "relative_rms": 0.0, + "divergence_details": [] + }, + { + "dtype": "float32", + "oracle": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "mode": "partial", + "cache_hit_tokens": 64, + "argmax_disagreements": [], + "max_abs_logit_difference": 4.2557716369628906e-05, + "relative_rms": 1.695055630079878e-06, + "divergence_details": [] + }, + { + "dtype": "float32", + "oracle": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "mode": "warm", + "cache_hit_tokens": 1984, + "argmax_disagreements": [], + "max_abs_logit_difference": 4.482269287109375e-05, + "relative_rms": 1.9181350126018515e-06, + "divergence_details": [] + }, + { + "dtype": "float32", + "oracle": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "mode": "warm_concurrent", + "cache_hit_tokens": 10240, + "argmax_disagreements": [], + "max_abs_logit_difference": 4.482269287109375e-05, + "relative_rms": 1.9181350126018515e-06, + "divergence_details": [] + } +] diff --git a/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-prefix-normal-allocation.json b/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-prefix-normal-allocation.json new file mode 100644 index 00000000..23497494 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-prefix-normal-allocation.json @@ -0,0 +1,346 @@ +[ + { + "dtype": "bfloat16", + "oracle": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "mode": "cold", + "cache_hit_tokens": 0, + "argmax_disagreements": [], + "max_abs_logit_difference": 0.0, + "relative_rms": 0.0, + "divergence_details": [] + }, + { + "dtype": "bfloat16", + "oracle": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "mode": "partial", + "cache_hit_tokens": 64, + "argmax_disagreements": [], + "max_abs_logit_difference": 0.5, + "relative_rms": 0.022843077778816223, + "divergence_details": [] + }, + { + "dtype": "bfloat16", + "oracle": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "mode": "warm", + "cache_hit_tokens": 1984, + "argmax_disagreements": [], + "max_abs_logit_difference": 0.515625, + "relative_rms": 0.023976627737283707, + "divergence_details": [] + }, + { + "dtype": "bfloat16", + "oracle": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "mode": "warm_concurrent", + "cache_hit_tokens": 10240, + "argmax_disagreements": [], + "max_abs_logit_difference": 0.515625, + "relative_rms": 0.023976627737283707, + "divergence_details": [] + }, + { + "dtype": "float32", + "oracle": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "mode": "cold", + "cache_hit_tokens": 0, + "argmax_disagreements": [], + "max_abs_logit_difference": 0.0, + "relative_rms": 0.0, + "divergence_details": [] + }, + { + "dtype": "float32", + "oracle": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "mode": "partial", + "cache_hit_tokens": 64, + "argmax_disagreements": [], + "max_abs_logit_difference": 4.2557716369628906e-05, + "relative_rms": 1.695055630079878e-06, + "divergence_details": [] + }, + { + "dtype": "float32", + "oracle": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "mode": "warm", + "cache_hit_tokens": 1984, + "argmax_disagreements": [], + "max_abs_logit_difference": 4.482269287109375e-05, + "relative_rms": 1.9181350126018515e-06, + "divergence_details": [] + }, + { + "dtype": "float32", + "oracle": [ + 9625, + 594, + 6722, + 374, + 12095, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610, + 40676, + 13, + 576, + 6303, + 3745, + 5610, + 6467, + 624, + 1986, + 374, + 4004, + 2266, + 13, + 576, + 2518, + 3745, + 5610 + ], + "mode": "warm_concurrent", + "cache_hit_tokens": 10240, + "argmax_disagreements": [], + "max_abs_logit_difference": 4.482269287109375e-05, + "relative_rms": 1.9181350126018515e-06, + "divergence_details": [] + } +] diff --git a/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-prefix-numerics.json b/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-prefix-numerics.json new file mode 100644 index 00000000..bb972b7d --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/qwen3-06b-prefix-numerics.json @@ -0,0 +1,56 @@ +[ + { + "dtype": "bfloat16", + "mode": "cold", + "cache_hit_tokens": 0, + "argmax_disagreements": [], + "max_abs_logit_difference": 0.0, + "relative_rms": 0.0, + "divergence_details": [] + }, + { + "dtype": "bfloat16", + "mode": "partial", + "cache_hit_tokens": 64, + "argmax_disagreements": [], + "max_abs_logit_difference": 0.5, + "relative_rms": 0.022843077778816223, + "divergence_details": [] + }, + { + "dtype": "bfloat16", + "mode": "warm", + "cache_hit_tokens": 1984, + "argmax_disagreements": [], + "max_abs_logit_difference": 0.515625, + "relative_rms": 0.023976627737283707, + "divergence_details": [] + }, + { + "dtype": "float32", + "mode": "cold", + "cache_hit_tokens": 0, + "argmax_disagreements": [], + "max_abs_logit_difference": 0.0, + "relative_rms": 0.0, + "divergence_details": [] + }, + { + "dtype": "float32", + "mode": "partial", + "cache_hit_tokens": 64, + "argmax_disagreements": [], + "max_abs_logit_difference": 4.2557716369628906e-05, + "relative_rms": 1.695055630079878e-06, + "divergence_details": [] + }, + { + "dtype": "float32", + "mode": "warm", + "cache_hit_tokens": 1984, + "argmax_disagreements": [], + "max_abs_logit_difference": 4.482269287109375e-05, + "relative_rms": 1.9181350126018515e-06, + "divergence_details": [] + } +] diff --git a/opt_prime/examples/pd/http_matrix/evidence/qwen3-4b-cache-fp32.json b/opt_prime/examples/pd/http_matrix/evidence/qwen3-4b-cache-fp32.json new file mode 100644 index 00000000..21a9baab --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/qwen3-4b-cache-fp32.json @@ -0,0 +1,757 @@ +{ + "model_name": "Qwen3-4B", + "prefix_cache": true, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.10.0+cu128", + "dtype": "float32", + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 2.6140184690011665, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 2.469158129999414, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 2.584197010000935, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 2.540827002027072, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 2.611479846003931, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 5.076289049000479, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 2.3999937229964416, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 2.540858873981051, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 2.8516252219851594, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 2.7277319990098476, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 4.914653153013205, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 4.986576478986535, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 5.028090634004911, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 5.06925764301559, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 4.820952296984615 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 2.448044166987529 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 7.136311356996885 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "after_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ] + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true, + "cold_baseline_equal": true, + "before_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "after_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ] + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Qwen3-4B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Qwen3-4B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Qwen3-4B\",finished_reason=\"length\"} 19\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Qwen3-4B\"} 44288\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Qwen3-4B\"} 608\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.04\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.06\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.08\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.25\"} 9\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 13\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.75\"} 13\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 13\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 15\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"5.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"7.5\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"10.0\"} 19\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:time_to_first_token_seconds_sum{model_name=\"Qwen3-4B\"} 19.15500577594503\nvllm:time_to_first_token_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.025\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.05\"} 5\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.075\"} 10\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 18\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.15\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.2\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.3\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.4\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.75\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 19\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:time_per_output_token_seconds_sum{model_name=\"Qwen3-4B\"} 1.2742696488398173\nvllm:time_per_output_token_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.8\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.5\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.0\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 7\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"5.0\"} 16\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"10.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"15.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"20.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"30.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"40.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"50.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"60.0\"} 19\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 19\nvllm:e2e_request_latency_seconds_sum{model_name=\"Qwen3-4B\"} 58.65736488997936\nvllm:e2e_request_latency_seconds_count{model_name=\"Qwen3-4B\"} 19\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Qwen3-4B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Qwen3-4B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"True\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Qwen3-4B\"} 44544\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Qwen3-4B\"} 35072\n", + "prefix_hit_tokens": 35072.0 +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/qwen3-4b-cache.json b/opt_prime/examples/pd/http_matrix/evidence/qwen3-4b-cache.json new file mode 100644 index 00000000..e4a50f31 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/qwen3-4b-cache.json @@ -0,0 +1,618 @@ +{ + "model_name": "Qwen3-4B", + "prefix_cache": true, + "passed": true, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": "2.10.0+cu128", + "device": "NVIDIA A100 80GB PCIe", + "cases": [ + { + "concurrency": 4, + "wall_s": 1.157064603001345, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 1.0982236790005118, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 1.066765146999387, + "baseline_equal": false + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 374, + 10161, + 369, + 279, + 6722, + 315, + 9625, + 13, + 6771, + 752, + 1744, + 13, + 358, + 1414, + 429, + 12095, + 374, + 279, + 6722, + 11 + ], + "e2e_s": 1.1289631230174564, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 1.1546258059970569, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 2.0710989810177125, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 0.8708334090188146, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 1.113788881979417, + "baseline_equal": false + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 374, + 10161, + 369, + 279, + 6722, + 315, + 9625, + 13, + 6771, + 752, + 1744, + 13, + 358, + 1414, + 429, + 12095, + 374, + 279, + 6722, + 11 + ], + "e2e_s": 2.0437719649926294, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 0.9625082009879407, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 1.0186384319968056, + "baseline_equal": false + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 374, + 10161, + 369, + 279, + 6722, + 315, + 9625, + 13, + 6771, + 752, + 1744, + 13, + 358, + 1414, + 429, + 12095, + 374, + 279, + 6722, + 11 + ], + "e2e_s": 1.9811588340089656, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 2.0088238760072272, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 2.064689474005718, + "baseline_equal": false + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 2.313285651995102 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 1.0522467579867225 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 374, + 10161, + 369, + 279, + 6722, + 315, + 9625, + 13, + 6771, + 752, + 1744, + 13, + 358, + 1414, + 429, + 12095, + 374, + 279, + 6722, + 11 + ], + "e2e_s": 2.419995628995821 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Qwen3-4B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Qwen3-4B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Qwen3-4B\",finished_reason=\"length\"} 17\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Qwen3-4B\"} 44032\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Qwen3-4B\"} 544\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.04\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.06\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.08\"} 5\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.25\"} 10\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 12\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.75\"} 12\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"5.0\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"7.5\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"10.0\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 17\nvllm:time_to_first_token_seconds_sum{model_name=\"Qwen3-4B\"} 7.092454752913909\nvllm:time_to_first_token_seconds_count{model_name=\"Qwen3-4B\"} 17\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.025\"} 3\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.05\"} 16\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.075\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.15\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.2\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.3\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.4\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.75\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 17\nvllm:time_per_output_token_seconds_sum{model_name=\"Qwen3-4B\"} 0.5323197415827082\nvllm:time_per_output_token_seconds_count{model_name=\"Qwen3-4B\"} 17\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.8\"} 2\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 4\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.5\"} 11\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.0\"} 13\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"5.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"10.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"15.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"20.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"30.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"40.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"50.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"60.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 17\nvllm:e2e_request_latency_seconds_sum{model_name=\"Qwen3-4B\"} 23.594366741977865\nvllm:e2e_request_latency_seconds_count{model_name=\"Qwen3-4B\"} 17\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Qwen3-4B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Qwen3-4B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"True\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Qwen3-4B\"} 44288\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Qwen3-4B\"} 34944\n", + "prefix_hit_tokens": 34944.0 +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/qwen3-4b-no-cache.json b/opt_prime/examples/pd/http_matrix/evidence/qwen3-4b-no-cache.json new file mode 100644 index 00000000..c3f4b73d --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/qwen3-4b-no-cache.json @@ -0,0 +1,614 @@ +{ + "model_name": "Qwen3-4B", + "prefix_cache": false, + "passed": true, + "cases": [ + { + "concurrency": 4, + "wall_s": 4.095043410983635, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 2.2055030329793226, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 2.9553901099716313, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 4.065600499976426, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 4.0910916150023695, + "baseline_equal": true + } + ] + }, + { + "concurrency": 8, + "wall_s": 7.584132659016177, + "rows": [ + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 2.0838619289861526, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 2.8289200870203786, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 5.143164925015299, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 5.232408629992278, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 5.976341186003992, + "baseline_equal": true + }, + { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 7.3540368099929765, + "baseline_equal": true + }, + { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 7.377973930008011, + "baseline_equal": true + }, + { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 7.575954277999699, + "baseline_equal": true + } + ] + } + ], + "baseline": { + "128": { + "input_tokens": 128, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315, + 9625, + 5267, + 16141, + 25, + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 14582, + 25, + 3555, + 374, + 279, + 6722, + 315 + ], + "e2e_s": 4.048442090017488 + }, + "2048": { + "input_tokens": 2048, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "e2e_s": 1.1545713389932644 + }, + "8192": { + "input_tokens": 8192, + "tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "e2e_s": 2.673155785014387 + } + }, + "cancel": { + "drained": true, + "subsequent_tokens_equal": true + }, + "disconnect": { + "drained": true, + "subsequent_tokens_equal": true + }, + "invalid_status": 400, + "remaining_requests": 0, + "metrics": "# HELP vllm:num_requests_running Requests executing on the stage.\n# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:num_requests_waiting Requests queued for the stage owner.\n# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:kv_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc{model_name=\"Qwen3-4B\"} 0.0\n# HELP vllm:gpu_cache_usage_perc Occupied fraction of the stage's KV lanes (1 = full).\n# TYPE vllm:gpu_cache_usage_perc gauge\nvllm:gpu_cache_usage_perc{model_name=\"Qwen3-4B\"} 0.0\n# HELP vllm:request_success_total Finished requests by finish reason.\n# TYPE vllm:request_success_total counter\nvllm:request_success_total{model_name=\"Qwen3-4B\",finished_reason=\"length\"} 17\n# HELP airm_stage_request_failures_total Requests that ended with an engine error.\n# TYPE airm_stage_request_failures_total counter\nairm_stage_request_failures_total{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:prompt_tokens_total Prompt tokens of finished requests.\n# TYPE vllm:prompt_tokens_total counter\nvllm:prompt_tokens_total{model_name=\"Qwen3-4B\"} 44032\n# HELP vllm:generation_tokens_total Generated tokens of finished requests.\n# TYPE vllm:generation_tokens_total counter\nvllm:generation_tokens_total{model_name=\"Qwen3-4B\"} 544\n# HELP vllm:time_to_first_token_seconds Time to first token (prefill stage: fenced prefill).\n# TYPE vllm:time_to_first_token_seconds histogram\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.001\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.005\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.02\"} 0\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.04\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.06\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.08\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.25\"} 4\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 6\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.75\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 8\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 9\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"5.0\"} 14\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"7.5\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"10.0\"} 17\nvllm:time_to_first_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 17\nvllm:time_to_first_token_seconds_sum{model_name=\"Qwen3-4B\"} 38.19619480409892\nvllm:time_to_first_token_seconds_count{model_name=\"Qwen3-4B\"} 17\n# HELP vllm:time_per_output_token_seconds Time per output token after the first (decode stage).\n# TYPE vllm:time_per_output_token_seconds histogram\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.025\"} 1\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.05\"} 9\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.075\"} 14\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 16\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.15\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.2\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.3\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.4\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.75\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 17\nvllm:time_per_output_token_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 17\nvllm:time_per_output_token_seconds_sum{model_name=\"Qwen3-4B\"} 0.9043958866114415\nvllm:time_per_output_token_seconds_count{model_name=\"Qwen3-4B\"} 17\n# HELP vllm:e2e_request_latency_seconds Stage request latency, from execution start to reply.\n# TYPE vllm:e2e_request_latency_seconds histogram\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.3\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 0\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.8\"} 1\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 2\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.5\"} 3\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.0\"} 3\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"2.5\"} 5\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"5.0\"} 11\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"10.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"15.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"20.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"30.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"40.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"50.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"60.0\"} 17\nvllm:e2e_request_latency_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 17\nvllm:e2e_request_latency_seconds_sum{model_name=\"Qwen3-4B\"} 66.2324672890536\nvllm:e2e_request_latency_seconds_count{model_name=\"Qwen3-4B\"} 17\n# HELP airm_kv_transfer_seconds KV transfer time from the prefill source to the decode destination.\n# TYPE airm_kv_transfer_seconds histogram\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.0005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.001\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.0025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.005\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.01\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.025\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.05\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.1\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.25\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"0.5\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"1.0\"} 0\nairm_kv_transfer_seconds_bucket{model_name=\"Qwen3-4B\",le=\"+Inf\"} 0\nairm_kv_transfer_seconds_sum{model_name=\"Qwen3-4B\"} 0.0\nairm_kv_transfer_seconds_count{model_name=\"Qwen3-4B\"} 0\n# HELP vllm:cache_config_info Information of the LLMEngine CacheConfig.\n# TYPE vllm:cache_config_info gauge\nvllm:cache_config_info{model_name=\"Qwen3-4B\",block_size=\"64\",num_gpu_blocks=\"576\",enable_prefix_caching=\"False\"} 1\n# HELP vllm:prefix_cache_queries_total Prompt tokens looked up in the prefix cache.\n# TYPE vllm:prefix_cache_queries_total counter\nvllm:prefix_cache_queries_total{model_name=\"Qwen3-4B\"} 44288\n# HELP vllm:prefix_cache_hits_total Prompt tokens served from the prefix cache.\n# TYPE vllm:prefix_cache_hits_total counter\nvllm:prefix_cache_hits_total{model_name=\"Qwen3-4B\"} 0\n" +} diff --git a/opt_prime/examples/pd/http_matrix/evidence/summary.json b/opt_prime/examples/pd/http_matrix/evidence/summary.json new file mode 100644 index 00000000..81114394 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence/summary.json @@ -0,0 +1,122 @@ +[ + { + "model": "qwen3-06b", + "mode": "bf16_no_cache", + "protocol_passed": true, + "equal_concurrent_outputs": 12, + "concurrent_outputs": 12, + "concurrency8_wall_s": 4.5096583620179445, + "prefix_hit_tokens": 0.0, + "file": "qwen3-06b-no-cache.json" + }, + { + "model": "qwen3-06b", + "mode": "bf16_cache", + "protocol_passed": true, + "equal_concurrent_outputs": 8, + "concurrent_outputs": 12, + "concurrency8_wall_s": 1.5730774590047076, + "prefix_hit_tokens": 34944.0, + "file": "qwen3-06b-cache.json" + }, + { + "model": "qwen3-06b", + "mode": "fp32_cache", + "protocol_passed": true, + "equal_concurrent_outputs": 12, + "concurrent_outputs": 12, + "concurrency8_wall_s": 4.691692629014142, + "prefix_hit_tokens": 35072.0, + "file": "qwen3-06b-cache-fp32.json" + }, + { + "model": "qwen3-4b", + "mode": "bf16_no_cache", + "protocol_passed": true, + "equal_concurrent_outputs": 12, + "concurrent_outputs": 12, + "concurrency8_wall_s": 7.584132659016177, + "prefix_hit_tokens": 0.0, + "file": "qwen3-4b-no-cache.json" + }, + { + "model": "qwen3-4b", + "mode": "bf16_cache", + "protocol_passed": true, + "equal_concurrent_outputs": 8, + "concurrent_outputs": 12, + "concurrency8_wall_s": 2.0710989810177125, + "prefix_hit_tokens": 34944.0, + "file": "qwen3-4b-cache.json" + }, + { + "model": "qwen3-4b", + "mode": "fp32_cache", + "protocol_passed": true, + "equal_concurrent_outputs": 12, + "concurrent_outputs": 12, + "concurrency8_wall_s": 5.076289049000479, + "prefix_hit_tokens": 35072.0, + "file": "qwen3-4b-cache-fp32.json" + }, + { + "model": "koni-8b", + "mode": "bf16_no_cache", + "protocol_passed": true, + "equal_concurrent_outputs": 12, + "concurrent_outputs": 12, + "concurrency8_wall_s": 7.984032834996469, + "prefix_hit_tokens": 0.0, + "file": "koni-8b-no-cache.json" + }, + { + "model": "koni-8b", + "mode": "bf16_cache", + "protocol_passed": true, + "equal_concurrent_outputs": 7, + "concurrent_outputs": 12, + "concurrency8_wall_s": 2.2306854949856643, + "prefix_hit_tokens": 35072.0, + "file": "koni-8b-cache-warm-cancel.json" + }, + { + "model": "koni-8b", + "mode": "fp32_cache", + "protocol_passed": true, + "equal_concurrent_outputs": 12, + "concurrent_outputs": 12, + "concurrency8_wall_s": 5.551692591019673, + "prefix_hit_tokens": 35072.0, + "file": "koni-8b-cache-fp32.json" + }, + { + "model": "gemma3-1b", + "mode": "bf16_no_cache", + "protocol_passed": true, + "equal_concurrent_outputs": 12, + "concurrent_outputs": 12, + "concurrency8_wall_s": 5.4193066009902395, + "prefix_hit_tokens": 0.0, + "file": "gemma3-1b-no-cache.json" + }, + { + "model": "gemma3-1b", + "mode": "bf16_cache", + "protocol_passed": true, + "equal_concurrent_outputs": 8, + "concurrent_outputs": 12, + "concurrency8_wall_s": 1.8549946299754083, + "prefix_hit_tokens": 34944.0, + "file": "gemma3-1b-cache.json" + }, + { + "model": "gemma3-1b", + "mode": "fp32_cache", + "protocol_passed": true, + "equal_concurrent_outputs": 12, + "concurrent_outputs": 12, + "concurrency8_wall_s": 4.413927393004997, + "prefix_hit_tokens": 35072.0, + "file": "gemma3-1b-cache-fp32.json" + } +] diff --git a/opt_prime/examples/pd/http_matrix/experiments/2026-10-23-http-matrix211.md b/opt_prime/examples/pd/http_matrix/experiments/2026-10-23-http-matrix211.md new file mode 100644 index 00000000..7011a13e --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/experiments/2026-10-23-http-matrix211.md @@ -0,0 +1,59 @@ +# Four-model GPU HTTP lifecycle and prefix-cache matrix + +- **Date**: 2026-10-23 +- **Task**: aicomp #211 (HTTP matrix), #212 (cache-history numerics), parent #201 +- **Host / device**: GPU host, A10080GB PCIe; each process sees one otherwise idle GPU; torch2.10.0+cu128, Transformers5.1.0 +- **Commit**: baseline21f2c87 (engine prerequisite), plus this branch's HTTP harness; no serving engine behavior change in this experiment + +## Question + +Do the four checkpoints serve valid HTTP requests under concurrent load, cancel/disconnect and reuse? Does prefix-cache history change token output? + +## Setup + +Actual serve_batched.build, BatchedOwner and BatchedService with loopback HTTP sockets. GPU paged cache,4 slots,9216 context,64-token pages,256-token prefill chunks. Inputs128/2048/8192 from the same repeated-context fixture as PR203;32 output tokens, temperature0, ignore_eos. Serial baselines, then mixed-length concurrency4 and8. Explicit cancellation and disconnect occur after streamed text is received; verify request removal, readiness and a subsequent request. Empty prompt must return400. No artificial transfer fence in this HTTP test. + +Revisions: Qwen3-0.6B c1899de289a04d12100db370d81485cdf75e47ca; Qwen3-4B1cfa9a7208912126459214e8b04321603b3df60c; KONI8B0172bf7e73977ae213613529f3d2d5400a33f678; Gemma3-1B5b11413a10db4e486ef16a20101fd028f8f2499c. Local model directories are copies of those snapshots; no new download. + +## Command + +```bash +CUDA_VISIBLE_DEVICES=0 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_HUB_DISABLE_PROGRESS_BARS=1 PYTHONPATH="$ENGINE/opt_prime" python "$ENGINE/opt_prime/examples/pd/http_matrix/validate.py" --model "$MODEL" --output result-no-cache.json +# Repeat for each model, then enable cache: +CUDA_VISIBLE_DEVICES=0 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_HUB_DISABLE_PROGRESS_BARS=1 PYTHONPATH="$ENGINE/opt_prime" python "$ENGINE/opt_prime/examples/pd/http_matrix/validate.py" --model "$MODEL" --prefix-cache --output result-cache.json +# Numerical control uses --dtype float32. The trace runner has the same CLI. +CUDA_VISIBLE_DEVICES=0 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_HUB_DISABLE_PROGRESS_BARS=1 PYTHONPATH="$ENGINE/opt_prime" python "$ENGINE/opt_prime/examples/pd/http_matrix/trace_prefix_http.py" --model Qwen/Qwen3-0.6B --prefix-cache --output qwen-cache-traced.json +``` + +Independent processes on the two GPUs overlap in wall time but never share a GPU. Local raw results/logs: analysis/http-matrix-211/. Client timing is end-to-end, not TTFT; server metrics include TTFT. Not a statistically powered capacity benchmark. + +## Results + +Cache disabled: all four models complete concurrency4/8, cancellation, disconnect, subsequent reuse and invalid-request checks. Each model's12 concurrent outputs exactly match its own serial HTTP baselines (48/48 combined). No remaining request IDs after drain. + +Cache enabled: lifecycle/count checks pass for Qwen3 sizes and Gemma; full concurrent output equality is8/12 each, with all discrepancies at2048 input. Gemma first difference24 is after its EOS and must not be treated as a changed completed answer. KONI first cancellation test fails because the subsequent warm-cache output differs from the cold baseline. This is retained as a failed run, not discarded. + +Cancellation control corrected: compare warmed state immediately before cancellation with immediately after, while separately recording equality to the original cold baseline. KONI passes both explicit cancellation and disconnect controls with identical before/after outputs; both differ from the cold baseline. Concurrent equality is7/12. Cache-hit counters are positive (34944 in original cases;35072 with the additional warm controls). + +## Numerical investigation (#212) + +Direct teacher-forced Qwen3-0.6B cold/64-hit/1984-hit and warm concurrent controls show BF16 logit differences up to0.515625, relativeRMS about2.4%; FP32 differences below4.5e-5. However, this direct diagnostic did not reproduce the actual HTTP argmax change and is insufficient alone to explain it. A separate allocation-mode control also did not reproduce that change. + +Actual HTTP trace reproduces it: partially cached2K prefill starts64 and uses256-token chunks; fully warmed prefill starts1984. At generated position1, partial-cache top logits for IDs374 and594 are both18.375, so argmax selects374. Warm-cache logits favor594 at18.5 over374 at18.25. Same input hash in both cases. This establishes a numerical tie in the observed HTTP path, not proof of equivalence for all cache histories/models. + +Actual HTTP FP32 controls finish on all four models: all lifecycle checks pass and48/48 concurrent outputs match each model's serial baseline. Cold-vs-warm128-token cancellation baselines also match in FP32. Qwen HTTP trace gives the previously tied IDs594/374 scores18.322538/18.220041 (partial) and18.322529/18.220039 (warm), consistently selecting594. The controls support numerical sensitivity as the observed cause; no rounding/tie-breaking change is made to the engine. + +Eight-request mixed-input burst wall times (seconds, one burst each, not confidence intervals): + +|Model|BF16 no cache|BF16 warm cache|FP32 warm cache| +|---|---:|---:|---:| +|Qwen3-0.6B|4.510|1.573|4.692| +|Qwen3-4B|7.584|2.071|5.076| +|KONI8B|7.984|2.231|5.552| +|Gemma3-1B|5.419|1.855|4.414| + +These are the same mixed128/2048/8192 prompt family with32 output tokens and four execution slots. Strong prefix reuse is intentional; do not extrapolate to cold traffic. BF16 cache configurations have reported raw output differences; FP32 is a correctness control, not a performance optimization. + +## Conclusion + +GPU HTTP lifecycle behavior now has real four-model evidence. Cache history and output precision require separate interpretation from cancellation correctness. Do not weaken greedy semantics or claim that all cache discrepancies are harmless. Remaining: finish numerical controls, ATOM HTTP matrix, RNGD HTTP/model routes, broader semantic prompts, cache/host-tier ownership and full-goal coverage. diff --git a/opt_prime/examples/pd/http_matrix/trace_prefix_http.py b/opt_prime/examples/pd/http_matrix/trace_prefix_http.py new file mode 100644 index 00000000..8fd1a0b8 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/trace_prefix_http.py @@ -0,0 +1,63 @@ +"""Record actual HTTP owner logits at the two initial 2K positions (#212).""" + +import hashlib +import json +import sys +from pathlib import Path + +import validate +from opt_prime.accel.pd import batching + +trace = {"samples": [], "prefills": []} +sample = batching.select_token +prefill = batching.ContinuousBatcher._prefill_chunk +build = validate.build + + +def select(logits, params, position): + if position in (2048, 2049): + top = logits.float().topk(5) + trace["samples"].append( + { + "position": position, + "ids": top.indices.tolist(), + "logits": top.values.tolist(), + } + ) + return sample(logits, params, position) + + +def chunk(self, r): + if len(r.request.input_ids) == 2048: + trace["prefills"].append( + { + "request": r.request.request_id, + "start": r.prefilled, + "end": min(r.prefilled + self.chunk, 2048), + "input_sha256": hashlib.sha256( + json.dumps([list(r.request.input_ids)]).encode() + ).hexdigest(), + } + ) + return prefill(self, r) + + +def traced_build(args): + owner = build(args) + m = owner.service.batcher.model + trace["attention"] = m.config._attn_implementation + trace["dtype"] = str(next(m.parameters()).dtype) + return owner + + +batching.select_token = select +batching.ContinuousBatcher._prefill_chunk = chunk +validate.build = traced_build +try: + validate.main() +finally: + batching.select_token = sample + batching.ContinuousBatcher._prefill_chunk = prefill + validate.build = build + output = Path(sys.argv[sys.argv.index("--output") + 1]) + output.with_suffix(".trace.json").write_text(json.dumps(trace, indent=2) + "\n") diff --git a/opt_prime/examples/pd/http_matrix/validate.py b/opt_prime/examples/pd/http_matrix/validate.py new file mode 100644 index 00000000..336b99dc --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/validate.py @@ -0,0 +1,250 @@ +"""Actual HTTP validation through serve_batched.build; one model/configuration per process.""" + +import argparse +import itertools +import json +import sys +import threading +import time +import urllib.error +import urllib.request +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from types import SimpleNamespace + +import torch +from transformers import AutoTokenizer + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from opt_prime.accel.serve.llmd import serve_stage +from serve_batched import build + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--model", required=True) + p.add_argument("--output", required=True) + p.add_argument("--prefix-cache", action="store_true") + p.add_argument("--dtype", choices=["bfloat16", "float32"], default="bfloat16") + a = p.parse_args() + torch.set_num_threads(4) + args = SimpleNamespace( + model=a.model, + backend="torch", + device="cuda:0", + dtype=a.dtype, + slots=4, + max_context=9216, + max_output=256, + kv="paged", + page_size=64, + chunk=256, + prefix_cache=a.prefix_cache, + kv_events=None, + kv_host_memory_gb=0, + reload_bandwidth=None, + prefill_rate=None, + remote_kv=None, + pd_prefill_device=None, + pd_prefill_python=None, + ) + result = { + "model_name": Path(a.model).name, + "prefix_cache": a.prefix_cache, + "passed": False, + "scope": "HTTP lifecycle and completion counts; numerical equality reported separately", + "torch": torch.__version__, + "dtype": a.dtype, + "device": torch.cuda.get_device_name(0), + "cases": [], + } + output = Path(a.output) + output.parent.mkdir(parents=True, exist_ok=True) + try: + owner = build(args) + except Exception as exc: + result["startup_error"] = repr(exc) + output.write_text(json.dumps(result, indent=2) + "\n") + raise + tok = AutoTokenizer.from_pretrained(a.model, local_files_only=True) + head = [tok.bos_token_id] if tok.bos_token_id is not None else [] + fill = tok.encode( + "This is background context. The red box contains apples. The blue box contains books.\n", + add_special_tokens=False, + ) + tail = tok.encode( + "\nQuestion: What is the capital of France?\nAnswer:", add_special_tokens=False + ) + prompts = {} + for n in (128, 2048, 8192): + left = n - len(head) - len(tail) + assert left >= 0 and fill + prompts[n] = head + (fill * ((left + len(fill) - 1) // len(fill)))[:left] + tail + assert len(prompts[n]) == n + server, server_thread = serve_stage(owner, "decode", 0) + url = f"http://127.0.0.1:{server.server_port}" + serial = itertools.count() + + def request(path, body=None, rid=None): + req = urllib.request.Request( + url + path, + data=None if body is None else json.dumps(body).encode(), + headers={ + "Content-Type": "application/json", + "x-request-id": rid or f"http211-{next(serial)}", + }, + ) + return urllib.request.urlopen(req, timeout=180) + + def body(n, count=32, stream=False): + return { + "model": a.model, + "prompt": prompts[n], + "max_tokens": count, + "temperature": 0, + "ignore_eos": True, + "stream": stream, + } + + def complete(n): + start = time.perf_counter() + with request("/v1/completions", body(n)) as response: + assert response.status == 200 + data = json.load(response) + tokens = data["choices"][0]["token_ids"] + assert len(tokens) == 32 and data["usage"]["completion_tokens"] == 32 + return { + "input_tokens": n, + "tokens": tokens, + "e2e_s": time.perf_counter() - start, + } + + def drained(rid, timeout=30): + end = time.monotonic() + timeout + while rid in owner.service.batcher.by_id: + if time.monotonic() > end: + raise TimeoutError(f"undrained request {rid}") + time.sleep(0.01) + + def client(): + try: + end = time.monotonic() + 30 + while not owner.started: + if time.monotonic() > end: + raise TimeoutError("owner startup") + time.sleep(0.01) + with request("/ready") as response: + assert response.status == 200 + baseline = {n: complete(n) for n in prompts} + result["baseline"] = baseline + for concurrency in (4, 8): + lengths = [(128, 2048, 8192)[i % 3] for i in range(concurrency)] + start = time.perf_counter() + with ThreadPoolExecutor(max_workers=concurrency) as pool: + rows = list(pool.map(complete, lengths)) + for row in rows: + row["baseline_equal"] = ( + row["tokens"] == baseline[row["input_tokens"]]["tokens"] + ) + result["cases"].append( + { + "concurrency": concurrency, + "wall_s": time.perf_counter() - start, + "rows": rows, + } + ) + # Explicit cancellation and socket disconnect after the first streamed token. + for mode in ("cancel", "disconnect"): + before_cancel = complete(128) + rid = f"http211-{mode}" + with request("/v1/completions", body(128, 256, True), rid) as response: + for raw in response: + if raw.startswith(b"data: ") and b"[DONE]" not in raw: + data = json.loads(raw[6:]) + if data.get("choices") and data["choices"][0].get("text"): + break + else: + raise AssertionError("stream had no text") + if mode == "cancel": + with request("/v1/cancel", {}, rid) as cancelled: + assert cancelled.status == 202 + drained(rid) + after = complete(128) + result[mode] = { + "drained": True, + "subsequent_tokens_equal": after["tokens"] + == before_cancel["tokens"], + "cold_baseline_equal": after["tokens"] == baseline[128]["tokens"], + "before_tokens": before_cancel["tokens"], + "after_tokens": after["tokens"], + } + assert result[mode]["subsequent_tokens_equal"], ( + "post-cancellation output changed" + ) + + try: + with request( + "/v1/completions", + {"model": a.model, "prompt": [], "max_tokens": 32}, + ): + raise AssertionError("invalid request accepted") + except urllib.error.HTTPError as exc: + assert exc.code == 400 + result["invalid_status"] = exc.code + with request("/ready") as response: + assert response.status == 200 + result["remaining_requests"] = len(owner.service.batcher.by_id) + assert result["remaining_requests"] == 0 + with request("/metrics") as response: + result["metrics"] = ( + response.read().decode().replace(a.model, result["model_name"]) + ) + hits = [ + float(line.rsplit(" ", 1)[1]) + for line in result["metrics"].splitlines() + if line.startswith("vllm:prefix_cache_hits_total{") + ] + assert hits, "prefix hit metric missing" + result["prefix_hit_tokens"] = sum(hits) + if a.prefix_cache: + assert result["prefix_hit_tokens"] > 0, ( + "cache enabled without observed hits" + ) + result["passed"] = True + except Exception as exc: # noqa: BLE001 - preserve failed hardware evidence + result["error"] = repr(exc) + finally: + for rid in list(owner.service.batcher.by_id): + owner.service.batcher.cancel(rid) + owner.submit("stop") + + thread = threading.Thread(target=client, daemon=True) + thread.start() + try: + owner.run() + except Exception as exc: # noqa: BLE001 - preserve failure and stop this process + result["engine_error"] = repr(exc) + finally: + thread.join(190) + server.shutdown() + server.server_close() + server_thread.join(5) + output.write_text(json.dumps(result, indent=2) + "\n") + if not result["passed"]: + raise RuntimeError( + result.get("error", result.get("engine_error", "validation failed")) + ) + print( + json.dumps( + { + k: v + for k, v in result.items() + if k not in ("cases", "baseline", "metrics") + } + ), + flush=True, + ) + + +if __name__ == "__main__": + main()