diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh index fc917052c..4efc84517 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh @@ -65,14 +65,17 @@ export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 CACHE_ARGS=() WARMUP_ARGS=() if require_agentic_kv_offload_backend hicache; then - # DeepSeek V4 HiCache currently rejects --hicache-size and supports - # capacity control only through a host/device token-capacity ratio. - # DSv4 exposes capacity as a host/device token ratio rather than bytes. - # Measurements put TP8 ratio=2 near 950 GB and TP4 ratio=8 near 1 TB, - # both below their configured capacities. The old TP4 ratio=16 - # used roughly 2 TB and violated the half-node allocation rule. + # DeepSeek V4 HiCache rejects --hicache-size and controls capacity only + # through a host/device token ratio, so TOTAL_CPU_DRAM_GB cannot apply + # directly. Host capacity scales with BOTH the ratio and device KV, so it + # also grows with mem-fraction-static -- the two knobs multiply. Measured: + # TP8 ratio=2 at mem-fraction 0.835 gives 999 GB. ratio=4 at mem-fraction + # 0.93 overshoots: it left only 5.84 GB free on a 2,964 GB node and the + # V4 paged pool failed to allocate. ratio=3 keeps the tier near 2 TB with + # room for the paged pool, page cache, AIPerf and the router, while still + # well above the old half-node rule that pinned TP8 to ratio=2. if [ "$TP" -ge 8 ]; then - DEFAULT_HICACHE_RATIO=2 + DEFAULT_HICACHE_RATIO=3 else DEFAULT_HICACHE_RATIO=8 fi @@ -116,21 +119,48 @@ if [ "$DP_ATTENTION" = "true" ]; then PARALLEL_ARGS+=( --dp "$TP" --tokenizer-worker-num "$TP" + --enable-prefill-delayer + # TEMP(validation): 5 instead of 20 for the A/B against #2701. + --prefill-decode-interval 5 --enable-dp-attention --enable-dp-attention-local-control-broadcast --incremental-streaming-output --stream-interval 20 --dist-init-addr "127.0.0.1:$((PORT + 2000))" --ep-size "$EP_SIZE" - --moe-runner-backend flashinfer_mxfp4 + --moe-a2a-backend megamoe + --enable-deepseek-v4-fp4-indexer --disable-flashinfer-autotune ) - MEM_FRACTION_STATIC=0.95 - if [ "$CONC" -ge 512 ]; then - # Leave room for FlashInfer's transient MoE workspace at the DEP8 tail. - MEM_FRACTION_STATIC=0.94 + # DEP4 shards the model over half the node, so per-rank weights roughly + # double and the weights-only floor rises above 0.9 (the engine reports a + # minimum viable 0.9013 and refuses to start). Keep upstream's 0.95 there. + # DEP8 has room for the lower value, which leaves mega-MoE workspace + # headroom. + if [ "$TP" -ge 8 ]; then + # Mega-MoE's transient workspace lives OUTSIDE the static allocation and + # needs a single ~7 GB contiguous block, so headroom must grow with + # concurrency. Measured at conc 256: 0.835 (~42 GB free) runs; 0.93 + # (~16 GB free) and 0.95 (~11 GB free) both die with a CUDA OOM on one + # DP rank, which then hangs the whole engine in the MLP-sync collective. + MEM_FRACTION_STATIC=0.93 + if [ "$CONC" -ge 512 ]; then + # TEMP(validation): 0.86, matching #2701's DEP8 conc512/576 tier. + MEM_FRACTION_STATIC=0.86 + elif [ "$CONC" -ge 384 ]; then + MEM_FRACTION_STATIC=0.89 + elif [ "$CONC" -ge 256 ]; then + MEM_FRACTION_STATIC=0.9 + fi + else + MEM_FRACTION_STATIC=0.95 fi - CHUNKED_PREFILL_SIZE=16384 + # --chunked-prefill-size is a GLOBAL budget: server_args.py divides it by + # dp_size, and dp_size is TP here. Scale it so every DEP shape gets the + # per-rank 8192 that was tuned, rather than 16384/rank at DEP4 -- which + # exceeds MegaMoE's per-rank token cap (a startup ValueError) and measured + # slower at DEP8 when tried directly. + CHUNKED_PREFILL_SIZE=$((8192 * TP)) else PARALLEL_ARGS+=( --moe-runner-backend flashinfer_mxfp4 @@ -147,9 +177,24 @@ MODEL_ARGS=( # AgentX concurrency counts live session trees, not individual requests. # Allow subagent fan-out to exceed CONC without clipping request bursts. MAX_RUNNING_REQUESTS=$((2 * CONC)) -CUDA_GRAPH_MAX_BS=$CONC +# Subagent fan-out means live requests exceed CONC (see MAX_RUNNING_REQUESTS +# above), so sizing decode graphs at CONC would drop every larger batch to +# eager decode. Capture past the fan-out; the runtime clamps this down to the +# request pool size anyway. +CUDA_GRAPH_MAX_BS=$((CONC * 4)) [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 +# --cuda-graph-max-bs is an alias whose dest is cuda_graph_max_bs_decode, so the +# two forms below are the same knob and must not both be passed. +CUDA_GRAPH_ARGS=(--cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS") +SWA_FULL_TOKENS_RATIO=0.1 +if [ "$DP_ATTENTION" = "true" ]; then + # Decode graphs must cover the padded MTP batch across all DP ranks, which + # exceeds CONC; capping at 64 would fall back to eager decode. + CUDA_GRAPH_ARGS=(--cuda-graph-max-bs-decode 544) + SWA_FULL_TOKENS_RATIO=0.075 +fi + export PYTHONNOUSERSITE=1 export TORCH_CUDA_ARCH_LIST=10.0 # Agentic warmup dispatches hundreds of large prompts at once. SGLang's @@ -168,6 +213,17 @@ export SGLANG_OPT_USE_JIT_NORM=1 export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1 export SGLANG_OPT_USE_TOPK_V2=1 export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1 +if [ "$DP_ATTENTION" = "true" ]; then + # MegaMoE's FP4/MXF4 activation path is opt-in -- both flags default False, + # so --moe-a2a-backend megamoe alone runs a different kernel than the one + # measured. DG_USE_FP4_ACTS / DG_USE_MXF4_KIND are forwarded to DeepGEMM + # automatically from these two. + export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1 + export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1 + # Must cover the per-rank prefill budget (8192) or startup raises; the + # extra 128 is headroom over the exact-fit boundary. + export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320 +fi if [ "${EVAL_ONLY}" != "true" ]; then export SGLANG_SIMULATE_ACC_LEN=2.49 export SGLANG_SIMULATE_ACC_METHOD=match-expected @@ -182,6 +238,23 @@ if [ -n "$TRITON_PTXAS_PATH" ]; then export TRITON_PTXAS_PATH echo "Using ptxas for Triton: $TRITON_PTXAS_PATH" fi +# TEMP(validation): apply the unified-cache load-back multipin fix +# (sgl-project/sglang#35880, cherry-pick of sgl-project/sglang#34975 onto the +# dev-nightly-0820 base) onto the image's editable sglang source before the +# server starts, so the conc-512 run does not die on the commit_load_back +# single-pin assertion. Drop this once the image includes the fix. +SGLANG_LOADBACK_PATCH="$SCRIPT_DIR/sglang-loadback-multipin.patch" +if [ -f "$SGLANG_LOADBACK_PATCH" ] && [ -d /sgl-workspace/sglang ]; then + # The runtime image ships git but not patch(1); git apply works on a + # plain (non-repo) source tree, which is how the image ships sglang. + if git -C /sgl-workspace/sglang apply --check "$SGLANG_LOADBACK_PATCH" 2>/dev/null; then + git -C /sgl-workspace/sglang apply "$SGLANG_LOADBACK_PATCH" \ + && echo "Applied sglang load-back multipin patch to /sgl-workspace/sglang" \ + || echo "ERROR: failed to apply sglang load-back multipin patch" + else + echo "sglang load-back multipin patch not applicable (already applied?), skipping" + fi +fi SGLANG_CMD=( "$SGLANG_PYTHON" -m sglang.launch_server --model-path "$MODEL_PATH" @@ -191,9 +264,9 @@ SGLANG_CMD=( --trust-remote-code "${PARALLEL_ARGS[@]}" --mem-fraction-static "$MEM_FRACTION_STATIC" - --swa-full-tokens-ratio 0.1 + --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO" --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" + "${CUDA_GRAPH_ARGS[@]}" --allow-auto-truncate --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" --tool-call-parser deepseekv4 diff --git a/benchmarks/single_node/agentic/sglang-loadback-multipin.patch b/benchmarks/single_node/agentic/sglang-loadback-multipin.patch new file mode 100644 index 000000000..aa0d201c4 --- /dev/null +++ b/benchmarks/single_node/agentic/sglang-loadback-multipin.patch @@ -0,0 +1,100 @@ +diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py +index faa14da3e..d2cfb422b 100644 +--- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py ++++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py +@@ -131,9 +131,14 @@ class UnifiedTreeNode: + self.id = UnifiedTreeNode.counter + UnifiedTreeNode.counter += 1 + self.write_through_pending_id: Optional[int] = None +- # Anchor NodeId of an in-flight H->D load-back reading this node's +- # host slots; such host copies must not be reclaimed until the ack. +- self.load_back_pending_id: Optional[int] = None ++ # Anchor NodeIds of in-flight H->D load-backs reading this node's ++ # host slots; such host copies must not be reclaimed until every ++ # anchor has acked. Multiple live anchors can pin one node: a ++ # descendant's Full-KV chain covers this node while another request ++ # anchors here for its independently-evicted aux (e.g. mamba) state. ++ # Overlapping transfers only READ the shared host slots and write ++ # disjoint destinations, so concurrent pins are safe to track. ++ self.load_back_pending_ids: set[int] = set() + + def component(self, component_type: ComponentType) -> ComponentData: + return self.component_data[component_type] +@@ -1081,7 +1086,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): + new_node.hit_count = child.hit_count + new_node.creation_time = child.creation_time + # Split fragments stay on the anchor's root path for the ack's walk. +- new_node.load_back_pending_id = child.load_back_pending_id ++ new_node.load_back_pending_ids = set(child.load_back_pending_ids) + + self._for_each_component_lru(child, UnifiedLRUList.remove_node) + +@@ -1192,7 +1197,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): + and cd.value is not None + and cd.host_value is not None + and node.write_through_pending_id is None +- and node.load_back_pending_id is None ++ and not node.load_back_pending_ids + ) + + def _for_each_component_lru( +@@ -1430,7 +1435,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): + return False + if ( + node.write_through_pending_id is not None +- or node.load_back_pending_id is not None ++ or node.load_back_pending_ids + ): + return False + return cd.host_lock_ref == 0 +@@ -1988,13 +1993,10 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): + for xfer in xfers: + for nid in xfer.nodes_to_load or (): + pinned = self.node_by_id(nid) +- # One live load-back per node; only the same anchor may +- # re-pin (a node can sit in Full and aux transfer lists). +- assert pinned.load_back_pending_id in (None, node_id), ( +- f"node {nid} pinned by load-back " +- f"{pinned.load_back_pending_id}, new anchor {node_id}" +- ) +- pinned.load_back_pending_id = node_id ++ # Multiple live load-backs may pin one node (set.add is ++ # also idempotent for a node sitting in both the Full and ++ # an aux transfer list of the same anchor). ++ pinned.load_back_pending_ids.add(node_id) + kv_xfer.device_indices = device_indices + self.components_by_type[BASE_COMPONENT_TYPE].commit_hicache_transfer( + node, +@@ -2025,10 +2027,12 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): + node = self.node_by_id(anchor_node_id) + while node is not None and node is not self.root_node: + if self.is_write_back: +- if node.load_back_pending_id != anchor_node_id: ++ if anchor_node_id not in node.load_back_pending_ids: + node = node.parent + continue +- node.load_back_pending_id = None ++ node.load_back_pending_ids.discard(anchor_node_id) ++ # The loaded copies become tracked duplicates only once the ++ # last in-flight load-back on this node acks. + self._update_duplicate_tracking(node) + node = node.parent + +@@ -2314,13 +2318,11 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): + # mark would pin the node's host copy against reclaim forever. + ongoing_load_ids = {node_id for _, node_id in ongoing_load_back} + for node in all_nodes: +- if ( +- node.load_back_pending_id is not None +- and node.load_back_pending_id not in ongoing_load_ids +- ): ++ stale_pins = node.load_back_pending_ids - ongoing_load_ids ++ if stale_pins: + E( +- f"[Ongoing] node {node.id} load_back_pending_id=" +- f"{node.load_back_pending_id} has no live load-back" ++ f"[Ongoing] node {node.id} load_back_pending_ids=" ++ f"{sorted(stale_pins)} have no live load-back" + ) + + if errors: diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 5d333e040..080040340 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1156,7 +1156,7 @@ dsv4-fp4-b300-sglang: - { tp: 8, ep: 8, dp-attn: true, conc-start: 4096, conc-end: 4096 } dsv4-fp4-b300-sglang-agentic-hicache-mtp: - image: lmsysorg/sglang:v0.5.17-cu130 + image: lmsysorg/sglang:dev-nightly-0820 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:b300-nv @@ -1165,21 +1165,22 @@ dsv4-fp4-b300-sglang-agentic-hicache-mtp: multinode: false scenarios: agentic-coding: - - dram-utilization: 0.80 + - dram-utilization: 0.95 + # TEMP(validation): sweep reduced to the single conc-512 point to validate + # the unified-cache load-back multipin patch applied by the recipe script. search-space: - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 20, 24, 32] } - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 40, 48, 52, 56, 60, 64, 72] } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [8, 16, 24, 32, 40, 64], router: { name: sglang-router, version: "0.3.2" } } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [32, 40, 48, 56, 64, 72, 80, 88, 96, 128], router: { name: sglang-router, version: "0.3.2" } } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [52, 72, 100, 128, 144, 196, 512], router: { name: sglang-router, version: "0.3.2" } } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [512], router: { name: sglang-router, version: "0.3.2" } } # DeepSeek-V4-Pro on B300 with EAGLE/MTP speculative decoding. Recipe is # selected inside benchmarks/single_node/dsv4_fp4_b300_sglang_mtp.sh by # DP_ATTENTION: # dp-attn: false -> TP-only + flashinfer_mxfp4 + chunked-prefill 8192 - # + EAGLE (3,1,4) + mem-fraction 0.90 - # dp-attn: true -> DP-attn + flashinfer_mxfp4 + chunked-prefill 32768 - # + EAGLE (1,1,2) + mem-fraction 0.92 + max-running 256 + # + mem-fraction 0.88 + swa-full-tokens-ratio 0.1 + # dp-attn: true -> DP-attn + megamoe + fp4 indexer + # + chunked-prefill 65536 + mem-fraction 0.90 + # + swa-full-tokens-ratio 0.075 + # + prefill-decode-interval 5 + # Both paths share EAGLE (3,1,4) and max-running-requests 2*CONC. dsv4-fp4-b300-sglang-mtp: image: lmsysorg/sglang:nightly-dev-cu13-20260610-f332e526 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index cfb0b4daa..3e2b4b9aa 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6325,3 +6325,44 @@ - "Expand the GB300 AgentX aggregate TP4 sweep to concurrency 1, 2, 4, 6, and 8." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2676 +- config-keys: + - dsv4-fp4-b300-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "Bump image from lmsysorg/sglang:v0.5.17-cu130 to lmsysorg/sglang:dev-nightly-0820 to pick up sgl-project/sglang#35017" + - "Add --prefill-decode-interval 10: +28% output throughput at conc 128 (2,433-2,470 -> 3,127-3,161 tok/s), closing the gap to the vLLM recipe from 1.32x to 1.03x. Under spec-decoding + DP attention SGLang synchronises decode globally, so a rank with no prefill work runs an idle batch and the busiest rank sets the clock for all eight; the interval bounds that." + - "Engine changes are scoped to the DP-attention path except the decode CUDA-graph cap. The TP-only path keeps flashinfer_mxfp4, chunked-prefill 8192, mem-fraction 0.88 and swa-full-tokens-ratio 0.1, because every measurement here ran with DP attention enabled." + - "Size the TP-only decode CUDA graphs at 4*CONC (still capped at 64) instead of CONC. AgentX concurrency counts live session trees, not requests, and the recipe already allows max-running-requests of 2*CONC for subagent fan-out -- so capturing only up to CONC dropped every larger decode batch to eager execution on exactly the low-concurrency rows that exist to measure latency. The runtime clamps the captured list to the request-pool size, so this cannot over-capture." + - "On the DP path, switch --moe-runner-backend flashinfer_mxfp4 to --moe-a2a-backend megamoe plus --enable-deepseek-v4-fp4-indexer, and export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1 and _USE_MXF4_KIND=1. Both env flags default to False, so the backend flag alone would select a different MegaMoE kernel than the one measured; DeepGEMM's DG_USE_FP4_ACTS / DG_USE_MXF4_KIND are forwarded from them automatically." + - "Scale chunked-prefill-size with dp_size (8192 * TP) instead of pinning 65536. server_args.py divides this global budget by dp_size, so a fixed 65536 gives the tuned 8192/rank at DEP8 but 16384/rank at DEP4 -- which exceeds MegaMoE's per-rank token cap and raises at startup, and which measured slower when tried directly at DEP8. Also export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320, since the 8192 default is an exact fit with no headroom." + - "On the DP path, replace --cuda-graph-max-bs with --cuda-graph-max-bs-decode 544 and add --enable-prefill-delayer. The generic flag aliases the same knob, and its CONC-capped value of 64 does not cover the padded MTP decode batch across DP ranks." + - "Scale DP-path mem-fraction-static down as concurrency rises: DEP8 uses 0.93 at conc 64/128, 0.9 at 256, 0.89 at 384 and 0.875 at 512/576; DEP4 keeps upstream's 0.95. MegaMoE's transient workspace is allocated outside the static budget and needs a single ~7 GB contiguous block, so the headroom left by mem-fraction-static has to grow with batch size. Measured at conc 256: 0.835 (~42 GB free) completes, while 0.93 (~16 GB free) and 0.95 (~11 GB free) both hit a CUDA OOM on a single DP rank -- which then hangs the whole engine, because the surviving ranks block forever in the MLP-sync collective and the client sees in-flight requests never return with zero errors. DEP4 cannot go lower: it shards the model over half the node, so per-rank weights roughly double and the weights-only floor rises above 0.9 (the engine reports a minimum viable 0.9013 and refuses to start). Also lower swa-full-tokens-ratio from 0.1 to 0.075." + - "Restructure the search space from 47 points to 12. Cut the TP-8 no-offload row to conc 1/4/8 and drop both TP-4 rows and the DEP4 no-offload row; keep DEP4+hicache at conc 32/48/64; and replace the DEP8 no-offload row with a DEP8+hicache row at conc 64/128/256/384/512/576. This tracks the vLLM agentic lane on the same runner, which sweeps DEP4 with a DRAM tier at 48/64 and DEP8 with a DRAM tier at 128/256/384/512/576. Measured at conc 256 with a CPU tier, SGLang 4,168 vs vLLM 4,199 tok/s." + - "Raise dram-utilization from 0.80 to 0.95, matching the vLLM agentic lane on the same runner, and the TP>=8 hicache ratio from 2 to 3 so the CPU tier reflects the allocated budget. DSv4 HiCache takes a host/device token ratio, not bytes, so the recipe cannot apply TOTAL_CPU_DRAM_GB directly, and host capacity scales with both the ratio and device KV -- meaning it also grows with mem-fraction-static. Measured at ratio 2 / mem-fraction 0.835 the tier allocates 999 GB (124.88 GB per rank across 8 ranks), a third of the 2,849 GB the vLLM agentic lane consumes. ratio 4 at mem-fraction 0.93 overshot the other way, leaving 5.84 GB free on a 2,964 GB node and failing to allocate the V4 paged pool; ratio 3 keeps the tier near 2 TB with room for the paged pool, page cache, AIPerf and the router. The TP<8 ratio is left at its existing value of 8." + - "Costs TTFT (p50 2.2x, p90 3.3x); AgentX is closed-loop so this is already priced into the throughput figures" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2704 + +- config-keys: + - dsv4-fp4-b300-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "TEMP(validation): reduce the agentic sweep to the single conc-512 point, and have the recipe script apply sglang-loadback-multipin.patch (sgl-project/sglang#35880, a cherry-pick of sgl-project/sglang#34975 onto the dev-nightly-0820 base, keeping the #34519 write-back gating) onto the image's editable sglang source before server start. This validates the fix for the unified_cache commit_load_back single-pin assertion (load_back_pending_id) that killed the conc-512 scheduler at PR #2701. Not intended for merge; revert once the image carries the fix." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2704 + +- config-keys: + - dsv4-fp4-b300-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "TEMP(validation): switch --prefill-decode-interval from 20 to 10 for an A/B against #2701 at the single conc-512 point, on top of the load-back multipin patch. Not intended for merge." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2704 + +- config-keys: + - dsv4-fp4-b300-sglang-agentic-hicache-mtp + scenario-type: + - agentic-coding + description: + - "TEMP(validation): switch --prefill-decode-interval from 10 to 5 and lower DEP8 conc-512 mem-fraction-static from 0.875 to 0.86 (matching #2701's latest tier) for the next A/B point at conc 512, on top of the load-back multipin patch. Not intended for merge." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2704