diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index 11e40a70eb..9f9dd0dcb8 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -66,7 +66,13 @@ export SGLANG_TIMEOUT_KEEP_ALIVE=900 # v1 dispatches to the precompiled HIP op in sgl-kernel (upstream MI355X CI # runs DSA models the same way). export SGLANG_OPT_USE_TOPK_V2=false - +# Optional quick-reduce quantisation (F-2 probe). Set in runner .env to activate; +# leave unset for baseline behaviour (falls back to NCCL/custom-AR). +# ROCM_QUICK_REDUCE_QUANTIZATION=INT4 — INT4 quick-reduce (tensor >= 16 MB for TP4/bf16) +# ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=1 — use faster fp16 kernels on ROCm (recommended with INT4) +[[ -n "${ROCM_QUICK_REDUCE_QUANTIZATION:-}" ]] && export ROCM_QUICK_REDUCE_QUANTIZATION +[[ -n "${ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16:-}" ]] && export ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16 + # HiCache L2 (host DRAM), optionally extended with Mooncake L3. # KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache or mooncake. # diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 0940c2495a..a40615ed4c 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1700,14 +1700,15 @@ minimaxm3-fp4-mi355x-vllm-agentic-mtp: - { tp: 2, kv-offloading: none, conc-list: [1, 2, 5], spec-decoding: mtp } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via SGLang with MTP speculative -# decoding. Two arms: -# (1) TP4/EP4 with HiCache KV offloading to DRAM, concurrency sweep [1, 2, 4, 8, 10, 12]; -# serves the throughput-dominant regime (conc ≥ 8). -# (2) TP8/EP1 without KV offloading, concurrency sweep [4, 10]; -# EP=1 eliminates the MoE all-to-all collective, reducing ITL at low-to-mid -# concurrency; GPU-resident KV keeps latency independent of DRAM bandwidth. -# c6 and c8 removed after sweep validation: dominated by TP4/EP4 arm. +# decoding. Three arms — two no-KV (MLA KV fits in HBM) plus one HiCache arm for +# the high-throughput regime: +# (1) TP4/EP4 no-KV: conc [1, 2, 4, 6, 8]; best interactivity in the low-mid range. +# (2) TP8/EP1 no-KV: conc [1, 2, 4]; EP=1 removes MoE all-to-all; best at low conc. +# (3) TP4/EP4 + HiCache: conc [1, 2, 4, 8, 10]; KV offload for throughput regime. # SA selects the Pareto-optimal arm per concurrency point. +# Validated on: runs 33724174688 (TP4/EP4 no-KV), 32986446019 (TP8/EP1 c4), +# 33489967280 (TP4/EP4 HiCache via PR#2777 sweep), +# 34103958942 (PR#2853 partial sweep). glm5.2-fp4-mi355x-sglang-agentic-mtp: image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 model: amd/GLM-5.2-MXFP4 @@ -1720,8 +1721,9 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10, 12], spec-decoding: mtp } - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 10], spec-decoding: mtp } + - { tp: 4, ep: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8], spec-decoding: mtp } + - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp } + - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10], spec-decoding: mtp } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via ATOM with MTP speculative # decoding. TP4 uses LMCache DRAM offload; TP8 is GPU-resident with no KV offload. # Recipe is from PR https://github.com/ROCm/ATOM/pull/1877 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9dfeb4a02a..e84ed58bc3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6957,7 +6957,18 @@ - "Tune the 8k/1k serving recipe with --mamba-full-memory-ratio 0.37 and --linear-attn-prefill-backend flashinfer; raise max-prefill-tokens and chunked-prefill-size from 16384 to 32768 and mem-fraction-static from 0.8 to 0.86." - "Expand the TP8 and TP4/EP1 sweep coverage through concurrency 640." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2866 - + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Extend the GLM-5.2 MI355X SGLang recipe to three arms covering both the interactivity and throughput regimes: two GPU-resident no-KV arms (MLA KV fits in HBM) and one TP4/EP4 HiCache arm for high concurrency." + - "TP4/EP4 no-KV arm, conc [1, 2, 4, 6, 8]: GPU-resident KV, best normalized interactivity at low-to-mid concurrency. Validated on run 33724174688 (norm_intvty_p90: c2=67.0, c6=66.3, c8=53.7)." + - "TP8/EP1 no-KV arm, conc [1, 2, 4]: EP=1 eliminates MoE all-to-all collective, best interactivity at c4. Validated on run 32986446019 (norm_intvty_p90=69.2, ITL p90=9.05 ms)." + - "TP4/EP4 + HiCache arm, conc [1, 2, 4, 8, 10]: DRAM KV offload for the throughput regime. Validated on run 33489967280 via PR#2777 sweep." + - "Add optional ROCM_QUICK_REDUCE_QUANTIZATION=INT4 / ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16 env-var hooks in the benchmark script (no-op when unset; enables F-2 quick-reduce probe without a script edit)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2853 - config-keys: - dsv4-fp4-b200-sglang-agentic-hicache-mtp