diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5.yaml new file mode 100644 index 000000000..b4b3d6511 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5.yaml @@ -0,0 +1,177 @@ +name: dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5 + +model: + path: nvidia/GLM-5.2-NVFP4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + request_plane: tcp + +health_check: + max_attempts: 270 + interval_seconds: 10 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 2 + decode_workers: 1 + gpus_per_decode: 8 + +backend: + type: trtllm + prefill_environment: &server_environment + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + TQDM_DISABLE: "1" + HF_HUB_DISABLE_PROGRESS_BARS: "1" + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: "1" + TRTLLM_WORKER_DISABLE_GC: "1" + TRTLLM_ENABLE_PDL: "1" + NCCL_GRAPH_MIXING_SUPPORT: "0" + MIMALLOC_PURGE_DELAY: "0" + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TRTLLM_SERVE_ENABLE_MSGSPEC: "1" + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: "0.10" + TRTLLM_KV_CACHE_TRANSFER_TIMEOUT_SEC: "600" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_RNDV_SCHEME: put_zcopy + TRTLLM_KVCACHE_SEND_BUFFER_COUNT: "1" + TRTLLM_KVCACHE_RECV_BUFFER_COUNT: "1" + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: "2.61" + DYN_TRTLLM_ENABLE_ATTENTION_DP: "1" + DYN_ENGINE_CONV_AFFINITY: "1" + DYN_TOKENIZER: fastokens + DYN_PUBLISH_KV_EVENTS: "0" + decode_environment: *server_environment + trtllm_config: + prefill: + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.75 + host_cache_size: 137438953472 + tokens_per_block: 64 + max_batch_size: 256 + max_num_tokens: 8192 + max_seq_len: 1048576 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_postprocess_workers: 8 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + tensor_parallel_size: 4 + decode: + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 5120 + max_tokens_in_buffer: 1048576 + kv_transfer_timeout_ms: 600000 + cuda_graph_config: + batch_sizes: [1] + enable_padding: true + enable_attention_dp: false + trust_remote_code: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + tokens_per_block: 64 + max_batch_size: 1 + max_num_tokens: 128 + max_seq_len: 1048576 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 1 + num_postprocess_workers: 4 + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + sparse_attention_config: + algorithm: dsa + enable_heuristic_topk: true + use_cute_dsl_paged_mqa_logits: true + use_cute_dsl_topk: true + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 8 + publish_events_and_metrics: false + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_ROUTER_TEMPERATURE: '0' + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_LOG: warn + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + args: + router-mode: kv + no-kv-events: true + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + orchestrator_placement: head + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES: "0" + SERVED_MODEL_NAME: GLM-5.2-NVFP4 + MAX_MODEL_LEN: '1048576' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + OPENAI_API_KEY: EMPTY + KV_OFFLOADING: dram diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 78fa30f65..113443eb2 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9151,7 +9151,7 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-agg-mtp: - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-agg-gb300-tp8-c1-b2-mtp8.yaml" - "SLURM_PARTITION=batch_3" # GLM-5.2 NVFP4 TensorRT-LLM AgentX on GB300 with Dynamo disaggregated serving. -# Six topology variants use NIXL KV transfer and MTP3/MTP5 decoding. +# Seven topology variants use NIXL KV transfer and MTP3/MTP5 decoding. glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc22.post1 model: nvidia/GLM-5.2-NVFP4 @@ -9167,6 +9167,23 @@ glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp: agentic-coding: - dram-utilization: 0.15239 search-space: + - spec-decoding: mtp + conc-list: [1] + kv-offloading: dram + kv-offload-backend: { name: native, version: "1.3.0rc22.post1" } + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=benchmarks/multi_node/srt-slurm-recipes/trtllm/glm5.2/gb300-fp4/agentic/dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5.yaml" + - "SLURM_PARTITION=batch_3" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false - spec-decoding: mtp conc-list: [30] kv-offloading: dram diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9fcda3df3..d06009855 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6476,7 +6476,6 @@ - "Add HiCache host-DRAM KV tier arms at TP4 concurrency 40, 48, 56, and 64 and TP2 concurrency 20, 24, 28, and 32, using hicache ratio 1.5 with write_through, direct io, and page_first_direct layout." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2693 - - config-keys: - kimik2.6-fp4-b200-dynamo-vllm - dsv4-fp4-b200-dynamo-vllm @@ -6547,3 +6546,14 @@ - "Filter AgentX traces at the same 202,752-token context limit used by both TileRT roles so oversized Weka trajectories are excluded before replay." - "Pin SemiAnalysisAI/srt-slurm PR #10 commit d1e6c97b3baf3e87103b6d83189544c3c7d61c38, stacked on the AMD/native-router PR #7 and base runtime PR #1, including explicit native HTTP dependencies, GLM-5.1-compatible Transformers v5 router tokenization, incomplete-snapshot recovery, backend-declared conversion GPU resources, pre-container NVIDIA driver-hook activation, and lossless Slurm container-environment exports." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2750 + +- config-keys: + - glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp + scenario-type: + - agentic-coding + description: + - "Add a GLM-5.2 concurrency-1 AgentX point using a 1-prefill/1-decode disaggregated topology with TP4 attention-DP prefill and TP8 decode." + - "Use MTP depth 5 with the committed golden acceptance target, tensor-parallel MoE on decode, and the CUTEDSL top-k path." + - "Align recipe KV offloading with the master config and keep the benchmark client colocated with the frontend orchestrator." + - "Add the new concurrency-1 topology to the existing disaggregated configuration key while retaining all published points." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2720