diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index 98b4531e4..ffbe16aae 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -72,25 +72,25 @@ export SGLANG_OPT_USE_TOPK_V2=false # # Per-arm L2 ratio (sizing rationale below) applies to both backends unless # overridden via HICACHE_RATIO. TP arm (182.7 GB/rank device pool): the -# working set oversubscribes the device pool ~3x at conc 32, so the host -# tier is what carries the radix hits - ratio 1.5 (~2.9 TB pinned incl. -# sidecars) validates through the conc-24 long-context storm for the -# mooncake arm. The DP-attention arm (159.4 GB/rank) only runs at conc >= -# 32, where each DP rank's ~8 sessions nearly fit in its own device pool -# (~1.5-1.6M of 1.7M tokens at conc 64) and the host tier just absorbs -# overflow - ratio 1.5 boots but the host OOM killer takes the server -# mid-storm at conc 48, so it runs ratio 0.5 (~1.2 TB pinned, ~1.8 TB of -# load headroom) at negligible hit-rate cost. The hicache-only arm has no -# L3 to fall back on, so these ratios are unvalidated there - override with -# HICACHE_RATIO if the host OOMs or hit-rate is poor. +# agentic-coding corpus saturates any fixed DRAM pool at conc ≥ 10; ratio 2.5 +# (~4.8 TB pinned) sustains throughput through conc 12 and leaves headroom for +# the mooncake arm's conc-24 long-context storm. The DP-attention arm +# (159.4 GB/rank) only runs at conc >= 32, where the host tier just absorbs +# overflow - ratio 0.5 (~1.2 TB pinned, ~1.8 TB of load headroom) at negligible +# hit-rate cost (ratio 1.5 OOMs the host mid-storm at conc 48). CACHE_ARGS=() if agentic_kv_offload_enabled; then if [ "$DP_ATTENTION" = "true" ]; then HICACHE_RATIO="${HICACHE_RATIO:-0.5}" else - HICACHE_RATIO="${HICACHE_RATIO:-1.5}" + # ratio=2.5 (vs the former 1.5): the agentic-coding corpus fills any + # fixed DRAM pool at conc ≥ 10; a larger host tier delays saturation + # and keeps throughput stable through conc 12. + HICACHE_RATIO="${HICACHE_RATIO:-2.5}" fi - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + # write_through_selective skips DRAM writes for non-reusable KV blocks, + # reducing host-bus traffic without affecting the cache hit rate. + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}" HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" case "$KV_OFFLOAD_BACKEND" in @@ -196,12 +196,13 @@ else CHUNKED_PREFILL_SIZE=32768 export AGENTIC_WARMUP_GRACE_PERIOD=3600 fi -MAX_RUNNING_REQUESTS=$((1 * CONC)) +# 2×CONC in-flight slots: MTP draft+verify transiently batches more tokens +# than CONC sessions; headroom prevents scheduler stalls under burst. +MAX_RUNNING_REQUESTS=$((2 * CONC)) [ "$MAX_RUNNING_REQUESTS" -gt 256 ] && MAX_RUNNING_REQUESTS=256 -CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS -# NOTE: with MTP num-steps=5 the draft+verify batch can momentarily exceed -# MAX_RUNNING_REQUESTS; if cuda-graph misses ("graph capture miss") appear in -# server.log under load, consider raising this to e.g. MAX_RUNNING_REQUESTS * 2. +# SGLang interpolates a bs list [1..max_bs] automatically; cap at 64 to +# keep graph-capture memory bounded without giving up coverage. +CUDA_GRAPH_MAX_BS=$(( MAX_RUNNING_REQUESTS < 64 ? MAX_RUNNING_REQUESTS : 64 )) if [ "${EVAL_ONLY:-false}" != "true" ]; then export SGLANG_SIMULATE_ACC_LEN=3.61 diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 200096d5c..40e6c75d9 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1699,9 +1699,13 @@ minimaxm3-fp4-mi355x-vllm-agentic-mtp: - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.3" }, conc-list: [32, 40], spec-decoding: mtp } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via SGLang with MTP speculative -# decoding. Two arms: (1) TP4/EP4 with HiCache KV offloading to DRAM, concurrency -# sweep [1, 2, 4, 8, 10, 12, 16]; (2) TP8/EP8 without KV offloading for low-latency -# comparison at conc [1, 2, 4]. SA selects the Pareto-optimal arm per concurrency. +# decoding. Two arms: +# (1) TP4/EP4 with HiCache KV offloading to DRAM, concurrency sweep [1, 2, 4, 8, 10, 12]; +# serves the throughput-dominant regime (conc ≥ 8). +# (2) TP8/EP1 without KV offloading, concurrency sweep [4, 6, 8, 10]; +# EP=1 eliminates the MoE all-to-all collective, reducing ITL at low-to-mid +# concurrency; GPU-resident KV keeps latency independent of DRAM bandwidth. +# SA selects the Pareto-optimal arm per concurrency point. glm5.2-fp4-mi355x-sglang-agentic-mtp: image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 model: amd/GLM-5.2-MXFP4 @@ -1714,8 +1718,8 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10, 12, 16], spec-decoding: mtp } - - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp } + - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10, 12], spec-decoding: mtp } + - { tp: 8, ep: 1, kv-offloading: none, conc-list: [4, 6, 8, 10], spec-decoding: mtp } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via ATOM with MTP speculative # decoding. TP4 uses LMCache DRAM offload; TP8 is GPU-resident with no KV offload. # Recipe is from PR https://github.com/ROCm/ATOM/pull/1877 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ab92e7921..9a709bee0 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6576,3 +6576,13 @@ - "Recipes sourced from srt-slurm (recipes/trtllm/qwen3.5-fp4/inferencex/gb300/{mtp,stp})." - "Runner: launch_gb300-nv.sh bumped from NVIDIA/srt-slurm@v1.0.29 to v1.0.72 for the dynamo-trt+qwen3.5+fp4 path." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2730 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Switch the TP8 arm from EP=8 to EP=1: removing the MoE all-to-all collective reduces ITL p50 by ~5% at conc 4 (6.95 ms vs 7.3 ms baseline) and raises interactivity P90 from 105 to 110.5 tok/s/user. Extend the TP8 concurrency sweep to [4, 6, 8, 10] to cover the mid-range crossover region." + - "Raise HiCache ratio from 1.5 to 2.5 and switch write policy to write_through_selective (per PR #2679): the agentic-coding corpus saturates any fixed DRAM pool at conc ≥ 10; the larger host tier sustains throughput through conc 12 (+12% tok/s/GPU vs conc 10). Trim the TP4 concurrency list to [1, 2, 4, 8, 10, 12] removing the untested conc 16 point." + - "Double MAX_RUNNING_REQUESTS to 2×CONC: MTP draft+verify transiently batches more tokens than CONC sessions; the extra headroom prevents scheduler stalls under burst. Cap CUDA_GRAPH_MAX_BS at 64 (SGLang auto-interpolates a batch-size list up to the cap)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2769