From b9691af7b86615fae859efdb92c63f3404032b61 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 24 Aug 2026 10:41:42 -0700 Subject: [PATCH 1/2] feat(config): add GLM-5.2 GB300 AgentX disaggregated point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 GLM-5.2 GB300 AgentX 并发度 1 的预填充与解码分离配置，并移除已替代的聚合配置。 --- ...namo-disagg-gb300-1p1d-tp8-c1-b1-mtp5.yaml | 177 ++++++++++++++++++ configs/nvidia-master.yaml | 43 ++--- perf-changelog.yaml | 11 ++ 3 files changed, 202 insertions(+), 29 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5.yaml new file mode 100644 index 000000000..b4b3d6511 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5.yaml @@ -0,0 +1,177 @@ +name: dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 2 + decode_workers: 1 + gpus_per_decode: 8 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.61" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1] + enable_padding: true + enable_attention_dp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + tokens_per_block: 64 + max_batch_size: 1 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 1 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + use_cute_dsl_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 8 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: head + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: dram diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f0842fe55..f7a00b8ff 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9156,9 +9156,9 @@ glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 16 dp-attn: true -# GLM-5.2 NVFP4 TensorRT-LLM AgentX on GB300 with Dynamo aggregated serving. -# One TP8 worker serves prefill and decode on the same eight GPUs at concurrency 1. -glm5.2-fp4-gb300-dynamo-trt-agentic-agg-mtp: +# GLM-5.2 NVFP4 TensorRT-LLM AgentX on GB300 with Dynamo disaggregated serving. +# Seven topology variants use NIXL KV transfer and MTP3/MTP5 decoding. +glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 model: nvidia/GLM-5.2-NVFP4 model-prefix: glm5.2 @@ -9166,45 +9166,30 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-agg-mtp: precision: fp4 framework: dynamo-trt router: { name: dynamo-router, version: "1.4.0.dev20260807" } + kv-p2p-transfer: nixl multinode: true - disagg: false + disagg: true scenarios: agentic-coding: - - search-space: + - dram-utilization: 0.15239 + search-space: - spec-decoding: mtp conc-list: [1] - kv-offloading: none + kv-offloading: dram + kv-offload-backend: { name: native, version: "1.3.0rc22.post1" } prefill: num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false + tp: 4 + ep: 4 + dp-attn: true additional-settings: - - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-agg-gb300-tp8-c1-b2-mtp8.yaml" + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5.yaml" - "SLURM_PARTITION=batch_3" decode: - num-worker: 0 + num-worker: 1 tp: 8 ep: 1 dp-attn: false - -# GLM-5.2 NVFP4 TensorRT-LLM AgentX on GB300 with Dynamo disaggregated serving. -# Six topology variants use NIXL KV transfer and MTP3/MTP5 decoding. -glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: - image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 - model: nvidia/GLM-5.2-NVFP4 - model-prefix: glm5.2 - runner: cluster:gb300-nv - precision: fp4 - framework: dynamo-trt - router: { name: dynamo-router, version: "1.4.0.dev20260807" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - agentic-coding: - - dram-utilization: 0.15239 - search-space: - spec-decoding: mtp conc-list: [30] kv-offloading: dram diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a9d2492a0..c9ef25911 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6387,3 +6387,14 @@ - "Replace the internal TileRT 0.1.5.post2+inferencex.1 queueing backport with the official 0.1.5.post3 PyPI wheel now that upstream has published it; remove the internal wheel builder and router patch" - "Bump the shared TileRT dependency and router metadata to 0.1.5.post3 for both fixed-seq-length and AgentX configurations while retaining the AgentX 1800-second queue timeout" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2650 + +- config-keys: + - glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - "Move the GLM-5.2 concurrency-1 AgentX point from an aggregated TP8 topology to a disaggregated 1-prefill/1-decode topology with TP4 attention-DP prefill and TP8 decode." + - "Use MTP depth 5 with the committed golden acceptance target, tensor-parallel MoE on decode, and the CUTEDSL top-k path." + - "Align recipe KV offloading with the master config and keep the benchmark client colocated with the frontend orchestrator." + - "Retire the superseded aggregated configuration key while retaining the existing disaggregated topology points." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From 524c19f6311c7dbb924535c8dbe2d1762ed71faf Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 24 Aug 2026 10:42:17 -0700 Subject: [PATCH 2/2] chore(changelog): link PR #2720 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在性能变更日志中补充 PR #2720 链接。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c9ef25911..a791507bf 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6397,4 +6397,4 @@ - "Use MTP depth 5 with the committed golden acceptance target, tensor-parallel MoE on decode, and the CUTEDSL top-k path." - "Align recipe KV offloading with the master config and keep the benchmark client colocated with the frontend orchestrator." - "Retire the superseded aggregated configuration key while retaining the existing disaggregated topology points." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2720