diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index 61add3eaf..2a32f7c9b 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -73,6 +73,9 @@ export SGLANG_DSV4_REASONING_EFFORT=high export SGLANG_USE_ROCM700A=0 export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton export AITER_BF16_FP8_MOE_BOUND=0 +# aiter batched GEMM for the absorbed MLA projections, carried by the v0.5.18 +# image and off by default in environ.py. +export SGLANG_OPT_USE_AITER_BATCHED_GEMM=1 # Unified radix tree: per-component (full-attn / SWA) cache management for # hybrid-attention models, plus proactive release of out-of-window SWA KV @@ -130,8 +133,10 @@ SGLANG_BACKEND_PORT="$PORT" # cap; same value the multi-node DeepSeek-V4-Pro-AgentX no_dp profile uses. CHUNKED_PREFILL_SIZE=8192 # MTP adds a draft KV pool and extra graph captures on top of the spec-none -# footprint, which ran at 0.90. -MEM_FRACTION_STATIC=0.85 +# footprint, which ran at 0.90. 0.89 recovers most of that: the DSv4 compressor +# state pools are sized from the full-attention pool and allocated after it, +# outside this budget, so the remainder has to stay large enough to cover them. +MEM_FRACTION_STATIC=0.89 PARALLEL_ARGS=(--tensor-parallel-size "$TP") if [ "$DP_ATTENTION" = "true" ]; then USE_SGLANG_ROUTER=true @@ -213,7 +218,7 @@ SGLANG_CMD=( --page-size 256 --swa-full-tokens-ratio 0.10 --kv-cache-dtype fp8_e4m3 - --disable-shared-experts-fusion + --enforce-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 67a060539..2195bad7e 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1716,18 +1716,9 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp } -# DeepSeek-V4-Pro FP4 agentic-coding benchmark on MI355X via SGLang with the -# in-checkpoint MTP head (EAGLE path, depth 3 -> golden AL 2.49 for -# thinking_on, golden_al_distribution/dsv4_mtp.yaml). Restores the single-node -# SGLang AgentX coverage removed in de493d859 (PR #2531, which deleted the -# spec-none dsv4-fp4-mi355x-sglang-agentic-hicache key and its script) so the -# aggregated SGLang path is comparable with dsv4-fp4-mi355x-vllm-agentic-mtp -# and dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp. The image matches the -# already-green disagg AgentX entry. Pure TP8 only: DSA + dp-attention hangs a -# collective under long-context prefill, so no DEP arm ships until that path is -# validated. conc 16 appears on both arms to isolate the host KV tier's gain. + dsv4-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260813 + image: lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260822 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -1738,5 +1729,5 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 2, 4, 8, 16], spec-decoding: mtp } + - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 2, 4, 8, 10], spec-decoding: mtp } - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 32, 48], spec-decoding: mtp } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a9d2492a0..877f87b74 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6387,3 +6387,15 @@ - "Replace the internal TileRT 0.1.5.post2+inferencex.1 queueing backport with the official 0.1.5.post3 PyPI wheel now that upstream has published it; remove the internal wheel builder and router patch" - "Bump the shared TileRT dependency and router metadata to 0.1.5.post3 for both fixed-seq-length and AgentX configurations while retaining the AgentX 1800-second queue timeout" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2650 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update image from lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260813 to lmsysorg/sglang-rocm:v0.5.18-rocm720-mi35x-20260822." + - "Enable SGLANG_OPT_USE_AITER_BATCHED_GEMM=1, the aiter batched GEMM for the absorbed MLA projections, which defaults off." + - "Raise mem-fraction-static from 0.85 to 0.89, enlarging the full-attention KV pool. The DSv4 compressor state pools are sized from that pool and allocated after it, outside this budget, so the remainder still has to cover them." + - "Switch --disable-shared-experts-fusion to --enforce-shared-experts-fusion, which fuses the single shared expert into the routed list. The tuned MoE tables in this image cover the resulting (inter_dim 384, expert 385, topk 7) shape at the token sizes this workload hits." + - "Split the parallelism by concurrency: add a TP4 arm at [1, 2, 4, 8, 10] with no host KV tier, where halving the GPUs per replica raises throughput per GPU, and remove the TP8 no-offload arm [1, 2, 4, 8, 16]. This drops the published TP8 concurrency 1, 2, 4, 8 and 16 points and adds TP4 concurrency 1, 2, 4, 8 and 10." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2713