From 526cd3c465d55b5433eeebe2b1db9122a16f3d8d Mon Sep 17 00:00:00 2001 From: Karan Verma Date: Sun, 23 Aug 2026 20:47:56 -0500 Subject: [PATCH 1/5] Refresh the DSv4 MI355X SGLang AgentX arm and split TP4/TP8 by concurrency Image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260813 -> lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260822. Serving flags: - enable SGLANG_OPT_USE_AITER_BATCHED_GEMM=1, the aiter batched GEMM for the absorbed MLA projections, which defaults off in environ.py - mem-fraction-static 0.85 -> 0.89 to enlarge the full-attention KV pool. The DSv4 compressor state pools are sized from that pool and allocated after it, outside this budget, so the remainder still has to cover them, which is why this stops short of 0.90. - --disable-shared-experts-fusion -> --enforce-shared-experts-fusion. The tuned MoE tables in this image cover the resulting (inter_dim 384, expert 385, topk 7) shape at the token sizes this workload hits. Search space, split by concurrency so the arms stop overlapping: - add TP4 at [1, 2, 4, 8, 10] with no host KV tier, where halving the GPUs per replica raises throughput per GPU - reduce the TP8 no-offload arm from [1, 2, 4, 8, 16] to [16] - net effect on published data: drops TP8 conc 1, 2, 4, 8 and adds TP4 conc 1, 2, 4, 8, 10; the hicache arm is unchanged at [16, 32, 48], and conc 16 still appears on both TP8 arms to isolate the host KV tier's gain TP4 doubles per-rank weights and leaves far less room for KV, which is why it stops at concurrency 10 and ships no hicache variant. runner (cluster:mi355x-amds) is unchanged and no directory structure changed. Also rewords the key's comment: "pure TP8 only" becomes "tensor-parallel only", which is what it meant, since the exclusion is DP-attention rather than TP4. Co-authored-by: Cursor --- .../agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 11 ++++++++--- configs/amd-master.yaml | 19 +++++++++++++------ perf-changelog.yaml | 12 ++++++++++++ 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index 61add3eafa..2a32f7c9b4 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -73,6 +73,9 @@ export SGLANG_DSV4_REASONING_EFFORT=high export SGLANG_USE_ROCM700A=0 export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton export AITER_BF16_FP8_MOE_BOUND=0 +# aiter batched GEMM for the absorbed MLA projections, carried by the v0.5.18 +# image and off by default in environ.py. +export SGLANG_OPT_USE_AITER_BATCHED_GEMM=1 # Unified radix tree: per-component (full-attn / SWA) cache management for # hybrid-attention models, plus proactive release of out-of-window SWA KV @@ -130,8 +133,10 @@ SGLANG_BACKEND_PORT="$PORT" # cap; same value the multi-node DeepSeek-V4-Pro-AgentX no_dp profile uses. CHUNKED_PREFILL_SIZE=8192 # MTP adds a draft KV pool and extra graph captures on top of the spec-none -# footprint, which ran at 0.90. -MEM_FRACTION_STATIC=0.85 +# footprint, which ran at 0.90. 0.89 recovers most of that: the DSv4 compressor +# state pools are sized from the full-attention pool and allocated after it, +# outside this budget, so the remainder has to stay large enough to cover them. +MEM_FRACTION_STATIC=0.89 PARALLEL_ARGS=(--tensor-parallel-size "$TP") if [ "$DP_ATTENTION" = "true" ]; then USE_SGLANG_ROUTER=true @@ -213,7 +218,7 @@ SGLANG_CMD=( --page-size 256 --swa-full-tokens-ratio 0.10 --kv-cache-dtype fp8_e4m3 - --disable-shared-experts-fusion + --enforce-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 67a0605398..b23307e1d7 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1722,12 +1722,18 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: # SGLang AgentX coverage removed in de493d859 (PR #2531, which deleted the # spec-none dsv4-fp4-mi355x-sglang-agentic-hicache key and its script) so the # aggregated SGLang path is comparable with dsv4-fp4-mi355x-vllm-agentic-mtp -# and dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp. The image matches the -# already-green disagg AgentX entry. Pure TP8 only: DSA + dp-attention hangs a -# collective under long-context prefill, so no DEP arm ships until that path is -# validated. conc 16 appears on both arms to isolate the host KV tier's gain. +# and dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp. The image is the +# v0.5.18 20260822 build, whose aiter batched-GEMM path this key enables via +# SGLANG_OPT_USE_AITER_BATCHED_GEMM. Tensor-parallel only: DSA + dp-attention +# hangs a collective under long-context prefill, so no DEP arm ships until that +# path is validated. +# TP4 owns the low-concurrency end, where halving the GPUs per replica raises +# throughput per GPU, and TP8 picks up from conc 16; conc 16 appears on both TP8 +# arms to isolate the host KV tier's gain. TP4 doubles per-rank weights, leaving +# far less room for KV, which is why it stops at conc 10 and carries no host +# tier. dsv4-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260813 + image: lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260822 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -1738,5 +1744,6 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 2, 4, 8, 16], spec-decoding: mtp } + - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 2, 4, 8, 10], spec-decoding: mtp } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [16], spec-decoding: mtp } - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 32, 48], spec-decoding: mtp } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 98eef2c543..db7c6f26c9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6378,3 +6378,15 @@ - "Serve the TP8 arm with FULL_AND_PIECEWISE cudagraphs, capturing the mixed prefill/decode batches as well as the uniform decode batches. The DEP8 arm is unchanged." - "Replace the TP8 concurrency list [1, 4, 8, 16, 24] with [1, 4, 8, 14], dropping the published concurrency 16 and 24 points and adding concurrency 14." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2707 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update image from lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260813 to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260822." + - "Enable SGLANG_OPT_USE_AITER_BATCHED_GEMM=1, the aiter batched GEMM for the absorbed MLA projections, which defaults off." + - "Raise mem-fraction-static from 0.85 to 0.89, enlarging the full-attention KV pool. The DSv4 compressor state pools are sized from that pool and allocated after it, outside this budget, so the remainder still has to cover them." + - "Switch --disable-shared-experts-fusion to --enforce-shared-experts-fusion, which fuses the single shared expert into the routed list. The tuned MoE tables in this image cover the resulting (inter_dim 384, expert 385, topk 7) shape at the token sizes this workload hits." + - "Split the parallelism by concurrency: add a TP4 arm at [1, 2, 4, 8, 10] with no host KV tier, where halving the GPUs per replica raises throughput per GPU, and reduce the TP8 no-offload arm from [1, 2, 4, 8, 16] to [16]. This drops the published TP8 concurrency 1, 2, 4 and 8 points and adds TP4 concurrency 1, 2, 4, 8 and 10." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From a6c357b0b01ea9105a23b0395699612dbdf65ded Mon Sep 17 00:00:00 2001 From: Karan Verma Date: Sun, 23 Aug 2026 22:09:17 -0500 Subject: [PATCH 2/5] Point the changelog entry at PR #2713 Co-authored-by: Cursor --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index db7c6f26c9..15ea2fba36 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6389,4 +6389,4 @@ - "Raise mem-fraction-static from 0.85 to 0.89, enlarging the full-attention KV pool. The DSv4 compressor state pools are sized from that pool and allocated after it, outside this budget, so the remainder still has to cover them." - "Switch --disable-shared-experts-fusion to --enforce-shared-experts-fusion, which fuses the single shared expert into the routed list. The tuned MoE tables in this image cover the resulting (inter_dim 384, expert 385, topk 7) shape at the token sizes this workload hits." - "Split the parallelism by concurrency: add a TP4 arm at [1, 2, 4, 8, 10] with no host KV tier, where halving the GPUs per replica raises throughput per GPU, and reduce the TP8 no-offload arm from [1, 2, 4, 8, 16] to [16]. This drops the published TP8 concurrency 1, 2, 4 and 8 points and adds TP4 concurrency 1, 2, 4, 8 and 10." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2713 From 6b790be1417c55584b73d4d8c1081e9797e85ae4 Mon Sep 17 00:00:00 2001 From: karverma-amd Date: Sun, 23 Aug 2026 22:11:57 -0500 Subject: [PATCH 3/5] Update amd-master.yaml --- configs/amd-master.yaml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index b23307e1d7..d7cad00971 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1716,22 +1716,7 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp } -# DeepSeek-V4-Pro FP4 agentic-coding benchmark on MI355X via SGLang with the -# in-checkpoint MTP head (EAGLE path, depth 3 -> golden AL 2.49 for -# thinking_on, golden_al_distribution/dsv4_mtp.yaml). Restores the single-node -# SGLang AgentX coverage removed in de493d859 (PR #2531, which deleted the -# spec-none dsv4-fp4-mi355x-sglang-agentic-hicache key and its script) so the -# aggregated SGLang path is comparable with dsv4-fp4-mi355x-vllm-agentic-mtp -# and dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp. The image is the -# v0.5.18 20260822 build, whose aiter batched-GEMM path this key enables via -# SGLANG_OPT_USE_AITER_BATCHED_GEMM. Tensor-parallel only: DSA + dp-attention -# hangs a collective under long-context prefill, so no DEP arm ships until that -# path is validated. -# TP4 owns the low-concurrency end, where halving the GPUs per replica raises -# throughput per GPU, and TP8 picks up from conc 16; conc 16 appears on both TP8 -# arms to isolate the host KV tier's gain. TP4 doubles per-rank weights, leaving -# far less room for KV, which is why it stops at conc 10 and carries no host -# tier. + dsv4-fp4-mi355x-sglang-agentic-mtp: image: lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260822 model: deepseek-ai/DeepSeek-V4-Pro From f49b2722ef00f672f75c5835f4bcbeb339716ab3 Mon Sep 17 00:00:00 2001 From: Thomas Wang <1am9trash@gmail.com> Date: Mon, 24 Aug 2026 18:57:39 +0800 Subject: [PATCH 4/5] Update amd-master.yaml --- configs/amd-master.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index d7cad00971..2195bad7e2 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1730,5 +1730,4 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: - dram-utilization: 0.80 search-space: - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 2, 4, 8, 10], spec-decoding: mtp } - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [16], spec-decoding: mtp } - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 32, 48], spec-decoding: mtp } From 58bbb80846cd671655860797aa35d38e26560ce6 Mon Sep 17 00:00:00 2001 From: Thomas Wang <1am9trash@gmail.com> Date: Mon, 24 Aug 2026 20:02:00 +0800 Subject: [PATCH 5/5] Update perf-changelog.yaml --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6af70ddb76..877f87b74c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6397,5 +6397,5 @@ - "Enable SGLANG_OPT_USE_AITER_BATCHED_GEMM=1, the aiter batched GEMM for the absorbed MLA projections, which defaults off." - "Raise mem-fraction-static from 0.85 to 0.89, enlarging the full-attention KV pool. The DSv4 compressor state pools are sized from that pool and allocated after it, outside this budget, so the remainder still has to cover them." - "Switch --disable-shared-experts-fusion to --enforce-shared-experts-fusion, which fuses the single shared expert into the routed list. The tuned MoE tables in this image cover the resulting (inter_dim 384, expert 385, topk 7) shape at the token sizes this workload hits." - - "Split the parallelism by concurrency: add a TP4 arm at [1, 2, 4, 8, 10] with no host KV tier, where halving the GPUs per replica raises throughput per GPU, and reduce the TP8 no-offload arm from [1, 2, 4, 8, 16] to [16]. This drops the published TP8 concurrency 1, 2, 4 and 8 points and adds TP4 concurrency 1, 2, 4, 8 and 10." + - "Split the parallelism by concurrency: add a TP4 arm at [1, 2, 4, 8, 10] with no host KV tier, where halving the GPUs per replica raises throughput per GPU, and remove the TP8 no-offload arm [1, 2, 4, 8, 16]. This drops the published TP8 concurrency 1, 2, 4, 8 and 16 points and adds TP4 concurrency 1, 2, 4, 8 and 10." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2713