From 2ceaf76c5cc8c71027067d7ecc8c9a8f988a5431 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Wed, 26 Aug 2026 23:06:34 -0400
Subject: [PATCH 01/12] qwen3.8next-fp4-b200-sglang-agentic-mtp: day-zero
 Qwen3.8-Flash-Next AgentX on B200
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add the first Qwen3.8-Flash-Next AgentX recipe on B200, served by SGLang with
native NEXTN MTP. SGLang is the plan-of-record engine for this model per
MODELS.md. Serves RadixArk/Qwen3.8-Flash-Next-NVFP4 at TP4 with modelopt_fp4
quantization, following the Qwen3.5 NVFP4 B200 sibling. Throughput runs pin an
interim acceptance length of 3.24 at three speculative tokens; eval-only runs
keep real target verification.

新增 B200 上首个 Qwen3.8-Flash-Next AgentX 配方，由 SGLang 以原生 NEXTN MTP
提供服务。按 MODELS.md，SGLang 是该模型的 PoR 引擎。使用
RadixArk/Qwen3.8-Flash-Next-NVFP4，TP4，modelopt_fp4 量化，参照 Qwen3.5 NVFP4
B200 同类配方。吞吐运行按 3 个投机 token 锁定临时接受长度 3.24，仅评测运行仍使用
真实目标验证。

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../qwen3.8next_fp4_b200_sglang_mtp.sh        | 192 ++++++++++++++++++
 configs/nvidia-master.yaml                    |  17 ++
 perf-changelog.yaml                           |  10 +
 3 files changed, 219 insertions(+)
 create mode 100755 benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh

diff --git a/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh
new file mode 100755
index 0000000000..23b1b3fcb1
--- /dev/null
+++ b/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh
@@ -0,0 +1,192 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -x
+
+# AgentX trace replay for Qwen3.8-Flash-Next NVFP4 on B200 with SGLang
+# native NEXTN MTP. Day-zero recipe; SGLang is the plan-of-record engine for
+# this model (MODELS.md). Throughput uses the golden synthetic AL; evals retain
+# real target-model verification.
+#
+# The checkpoint is RadixArk/Qwen3.8-Flash-Next-NVFP4 (126 GiB,
+# quantization_config.quant_method = modelopt), so --quantization modelopt_fp4
+# matches the same flag the Qwen3.5 NVFP4 sibling uses. The model ships native
+# MTP modules (kept unquantized by the checkpoint's ignore list), so NEXTN
+# needs no external drafter.
+
+source "$(dirname "$0")/../../benchmark_lib.sh"
+
+# Use the lightweight GSM8K eval instead of the AgentX SWE-bench default.
+export EVAL_FRAMEWORK="lm-eval"
+
+check_env_vars \
+    MODEL TP CONC EP_SIZE KV_OFFLOADING \
+    TOTAL_CPU_DRAM_GB RESULT_DIR DURATION
+
+SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-10}
+
+if [[ -n "${SLURM_JOB_ID:-}" ]]; then
+    echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
+fi
+
+if [[ -n "${MODEL_PATH:-}" ]]; then
+    if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
+        hf download "$MODEL" --local-dir "$MODEL_PATH"
+    fi
+else
+    hf download "$MODEL"
+    export MODEL_PATH="$MODEL"
+fi
+nvidia-smi
+
+export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126_256k
+resolve_trace_source
+install_agentic_deps
+
+SERVER_LOG="$RESULT_DIR/server.log"
+mkdir -p "$RESULT_DIR"
+
+CACHE_ARGS=()
+if require_agentic_kv_offload_backend hicache; then
+    REQUESTED_HICACHE_TOTAL_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-$TOTAL_CPU_DRAM_GB}"
+    if [ "$REQUESTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then
+        echo "Error: requested HiCache pool ${REQUESTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2
+        exit 1
+    fi
+    TOTAL_CPU_DRAM_GB="$REQUESTED_HICACHE_TOTAL_GB"
+    # SGLang applies --hicache-size independently to Qwen's target KV and
+    # Mamba pools. Native NEXTN also creates a draft KV pool with the same
+    # slot count; its one attention layer adds 1/15 of the target KV bytes.
+    # Reserve 1 GB/rank for page alignment and enforce H * 31/15 per rank.
+    HICACHE_ALIGNMENT_RESERVE_GB=$TP
+    HICACHE_USABLE_TOTAL_GB=$((TOTAL_CPU_DRAM_GB - HICACHE_ALIGNMENT_RESERVE_GB))
+    if [ "$HICACHE_USABLE_TOTAL_GB" -lt 1 ]; then
+        echo "Error: insufficient DRAM after HiCache alignment reserve" >&2
+        exit 1
+    fi
+    MAX_HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB * 15 / TP / 31))
+    HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}"
+    if [ "$HICACHE_SIZE_GB" -lt 1 ] || [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then
+        echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB outside 1..$MAX_HICACHE_SIZE_GB" >&2
+        exit 1
+    fi
+    PROJECTED_HICACHE_TOTAL_GB=$(((HICACHE_SIZE_GB * TP * 31 + 14) / 15 + HICACHE_ALIGNMENT_RESERVE_GB))
+    if [ "$PROJECTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then
+        echo "Error: projected HiCache use ${PROJECTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2
+        exit 1
+    fi
+    echo "HiCache CPU pools: ${HICACHE_SIZE_GB} GB target + Mamba + 1/15 draft per rank across TP=${TP}; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB <= ${TOTAL_CPU_DRAM_GB} GB"
+    CACHE_ARGS=(
+        --page-size 64
+        --enable-hierarchical-cache
+        --hicache-size "$HICACHE_SIZE_GB"
+        --hicache-io-backend kernel
+        --hicache-mem-layout page_first
+        --hicache-write-policy write_through_selective
+    )
+fi
+
+PARALLEL_ARGS=(
+    --tp "$TP"
+    --dp 1
+    --ep-size "$EP_SIZE"
+)
+
+# TP4 needs parallel tokenization to keep 256k AgentX warmups below the client
+# request timeout. Keep TP2 on SGLang's single-worker default: multi-tokenizer
+# startup races with the TP2 HiCache shared-memory initialization path.
+TOKENIZER_ARGS=()
+if [ "$TP" -ge 4 ]; then
+    TOKENIZER_ARGS=(--tokenizer-worker-num 6)
+fi
+
+# AgentX concurrency counts live session trees rather than individual HTTP
+# requests. Leave room for subagent fan-out and avoid spending HBM on graphs
+# above the batch sizes that remain useful for this long-context workload.
+MAX_RUNNING_REQUESTS=$((2 * CONC))
+CUDA_GRAPH_MAX_BS="$CONC"
+[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64
+
+export TORCH_CUDA_ARCH_LIST="10.0"
+export PYTHONNOUSERSITE=1
+export NCCL_NVLS_ENABLE=1
+export SGL_ENABLE_JIT_DEEPGEMM=false
+export SGLANG_ENABLE_FLASHINFER_GEMM=true
+# Keep server-side connections alive beyond AIPerf's 300-second client pool
+# timeout so bursty AgentX trajectories cannot reuse a closing idle socket.
+export SGLANG_TIMEOUT_KEEP_ALIVE=1800
+
+if [ "${EVAL_ONLY:-false}" != "true" ]; then
+    # Qwen3.8-Flash-Next acceptance length, SPEED-Bench coding.
+    # --speculative-num-steps 3 with 4 draft tokens is 3 speculative
+    # tokens per verification step, i.e. the MTP=3 cell -> AL 3.24.
+    # Measured thinking=off in speedbench-al run 33031708148; the
+    # thinking=on collection is still in flight, so this is an
+    # interim value and is not yet a committed golden_al_distribution
+    # curve. Refresh once qwen3.8next_mtp.yaml lands.
+    export SGLANG_SIMULATE_ACC_LEN=3.24
+    export SGLANG_SIMULATE_ACC_METHOD=match-expected
+    export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token
+fi
+
+SGLANG_CMD=(
+    python3 -m sglang.launch_server
+    --model-path "$MODEL_PATH"
+    --served-model-name "$MODEL"
+    --host 0.0.0.0
+    --port "$PORT"
+    --trust-remote-code
+    "${PARALLEL_ARGS[@]}"
+    --enable-symm-mem
+    --quantization modelopt_fp4
+    --fp4-gemm-backend flashinfer_cutlass
+    --kv-cache-dtype fp8_e4m3
+    --mamba-ssm-dtype bfloat16
+    --attention-backend trtllm_mha
+    --moe-runner-backend flashinfer_trtllm
+    --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS"
+    --max-running-requests "$MAX_RUNNING_REQUESTS"
+    --max-prefill-tokens 16384
+    --chunked-prefill-size 16384
+    --mem-fraction-static 0.80
+    --stream-interval 50
+    --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL"
+    "${TOKENIZER_ARGS[@]}"
+    --tokenizer-path "$MODEL"
+    --reasoning-parser qwen3
+    --tool-call-parser qwen3_coder
+    --speculative-algorithm NEXTN
+    --speculative-num-steps 3
+    --speculative-eagle-topk 1
+    --speculative-num-draft-tokens 4
+    --enable-metrics
+    --enable-cache-report
+    "${CACHE_ARGS[@]}"
+)
+
+printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt"
+printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt"
+"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 &
+SERVER_PID=$!
+
+capture_cache_metrics() {
+    {
+        echo "=== SGLang cache metrics snapshot $(date --iso-8601=seconds) ==="
+        curl -fsS "http://localhost:$PORT/metrics" 2>/dev/null \
+            | grep -E '^(sglang:(cache_hit_rate|cached_tokens_total|prompt_tokens_total|hicache_host_used_tokens|hicache_host_total_tokens|token_usage|num_requests_running|num_requests_waiting))' \
+            || true
+        echo "============================================================"
+    } >> "$SERVER_LOG"
+}
+
+wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+
+capture_cache_metrics
+trap capture_cache_metrics EXIT
+
+if [ "${EVAL_ONLY:-false}" = "true" ]; then
+    run_eval --port "$PORT"
+else
+    build_replay_cmd "$RESULT_DIR"
+    REPLAY_CMD+=" --server-metrics http://localhost:$PORT/metrics"
+    run_agentic_replay_and_write_outputs "$RESULT_DIR"
+fi
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 438a3e9da0..7ad5c54851 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -7392,6 +7392,23 @@ qwen3.5-fp4-b200-sglang-agentic-mtp:
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 14] }
       - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 18, 20, 22, 24, 28, 32] }
 
+
+# Qwen3.8-Flash-Next NVFP4 AgentX on B200 via SGLang with native NEXTN MTP.
+# Day-zero recipe: SGLang is the plan-of-record engine for this model. TP4 at
+# 126 GiB of NVFP4 weights leaves ample HBM for the 256k-capped agentic traces.
+qwen3.8next-fp4-b200-sglang-agentic-mtp:
+  image: lmsysorg/sglang:qwen38flashnext
+  model: RadixArk/Qwen3.8-Flash-Next-NVFP4
+  model-prefix: qwen3.8next
+  runner: cluster:b200-nscale
+  precision: fp4
+  framework: sglang
+  multinode: false
+  scenarios:
+    agentic-coding:
+    - dram-utilization: 0.8
+      search-space:
+      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] }
 qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp:
   image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b
   model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index ab92e7921c..4b17f4ceb0 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6576,3 +6576,13 @@
     - "Recipes sourced from srt-slurm (recipes/trtllm/qwen3.5-fp4/inferencex/gb300/{mtp,stp})."
     - "Runner: launch_gb300-nv.sh bumped from NVIDIA/srt-slurm@v1.0.29 to v1.0.72 for the dynamo-trt+qwen3.5+fp4 path."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2730
+
+- config-keys:
+    - qwen3.8next-fp4-b200-sglang-agentic-mtp
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Add the day-zero Qwen3.8-Flash-Next NVFP4 AgentX recipe on B200 with SGLang native NEXTN MTP at TP4 and concurrency 1/4/8/12/16."
+    - "Serve RadixArk/Qwen3.8-Flash-Next-NVFP4 with modelopt_fp4 quantization, the trtllm_mha attention backend, and the flashinfer_trtllm MoE runner, following the Qwen3.5 NVFP4 B200 sibling."
+    - "Pin throughput runs to an interim acceptance length of 3.24 measured at three speculative tokens; eval-only runs keep real target verification."
+  pr-link: TBD

From a7bfe5f5c50a7ca39e94af5961d0b7ad230852aa Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Wed, 26 Aug 2026 23:06:43 -0400
Subject: [PATCH 02/12] =?UTF-8?q?Fill=20perf-changelog=20pr-link=20for=20#?=
 =?UTF-8?q?2751=20/=20=E8=A1=A5=E5=85=A8=20#2751=20=E7=9A=84=20perf-change?=
 =?UTF-8?q?log=20pr-link?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 perf-changelog.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 4b17f4ceb0..800143ea7a 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6585,4 +6585,4 @@
     - "Add the day-zero Qwen3.8-Flash-Next NVFP4 AgentX recipe on B200 with SGLang native NEXTN MTP at TP4 and concurrency 1/4/8/12/16."
     - "Serve RadixArk/Qwen3.8-Flash-Next-NVFP4 with modelopt_fp4 quantization, the trtllm_mha attention backend, and the flashinfer_trtllm MoE runner, following the Qwen3.5 NVFP4 B200 sibling."
     - "Pin throughput runs to an interim acceptance length of 3.24 measured at three speculative tokens; eval-only runs keep real target verification."
-  pr-link: TBD
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2751

From 14f0e732490838981dcd0afbb41152dea052ff4e Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Wed, 26 Aug 2026 23:20:42 -0400
Subject: [PATCH 03/12] Route qwen3.8next fp4 in the B200 nscale launcher
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

All six B200 cells failed in one minute with "Unsupported model prefix/
precision: qwen3.8next/fp4". launch_b200-nscale-compat.sh resolves every
model-prefix + precision pair to a pre-staged directory under /scratch/models
through a hardcoded if-chain, and qwen3.8next had no branch, so it fell to the
else and exited 1 before Slurm was ever touched.

Add the branch, following the dsv4 pattern: prefer an explicitly supplied
MODEL_PATH, then the first staged candidate directory, so a differently named
staging dir needs no further code change. This launcher bind-mounts MODEL_PATH
into the container and exports MODEL=$MODEL_PATH, so unlike the B300, H200 and
MI355X paths there is no hf-download fallback and the checkpoint has to be
staged on the nscale cluster for the sweep to pass.

B200 的全部 6 个单元在一分钟内失败，报错 "Unsupported model prefix/precision:
qwen3.8next/fp4"。launch_b200-nscale-compat.sh 通过硬编码的 if 链把每组
model-prefix + precision 解析到 /scratch/models 下预置的权重目录，其中没有
qwen3.8next 分支，因此落入 else 并在进入 Slurm 之前即以 1 退出。

参照 dsv4 的写法补上该分支：优先使用显式提供的 MODEL_PATH，其次取第一个存在的
候选目录，这样即使权重目录换名也无需再改代码。该 launcher 会把 MODEL_PATH
绑定挂载进容器并导出 MODEL=$MODEL_PATH，因此与 B300、H200、MI355X 路径不同，
它没有 hf 下载兜底，权重必须预先在 nscale 集群上就位，sweep 才可能通过。

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 perf-changelog.yaml                  |  1 +
 runners/launch_b200-nscale-compat.sh | 19 +++++++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 800143ea7a..5741adb625 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6585,4 +6585,5 @@
     - "Add the day-zero Qwen3.8-Flash-Next NVFP4 AgentX recipe on B200 with SGLang native NEXTN MTP at TP4 and concurrency 1/4/8/12/16."
     - "Serve RadixArk/Qwen3.8-Flash-Next-NVFP4 with modelopt_fp4 quantization, the trtllm_mha attention backend, and the flashinfer_trtllm MoE runner, following the Qwen3.5 NVFP4 B200 sibling."
     - "Pin throughput runs to an interim acceptance length of 3.24 measured at three speculative tokens; eval-only runs keep real target verification."
+    - "Route the qwen3.8next fp4 prefix to its staged checkpoint in the B200 nscale launcher, which previously rejected the prefix outright."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2751
diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh
index 6621bc8a83..1af088fa0e 100644
--- a/runners/launch_b200-nscale-compat.sh
+++ b/runners/launch_b200-nscale-compat.sh
@@ -92,6 +92,25 @@ elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" ]]; then
 elif [[ $MODEL_PREFIX == "kimik3" && $PRECISION == "fp4" ]]; then
     export MODEL_PATH="/scratch/models/Kimi-K3"
     export SRT_SLURM_MODEL_PREFIX="kimik3"
+elif [[ $MODEL_PREFIX == "qwen3.8next" && $PRECISION == "fp4" ]]; then
+    # Qwen3.8-Flash-Next NVFP4. Like the dsv4 branch, prefer an explicitly
+    # supplied MODEL_PATH, then a staged directory, so a differently named
+    # staging dir does not need a code change here. This launcher bind-mounts
+    # MODEL_PATH into the container and exports MODEL=$MODEL_PATH, so there is
+    # no hf-download fallback: the checkpoint must be staged on the cluster.
+    SELECTED_MODEL_PATH=""
+    if [[ -n "${MODEL_PATH:-}" && -d "${MODEL_PATH}" ]]; then
+        SELECTED_MODEL_PATH="$MODEL_PATH"
+    else
+        for candidate in /scratch/models/Qwen3.8-Flash-Next-NVFP4 /scratch/models/Qwen3.8-Flash-Next-FP8 /scratch/models/Qwen3.8-Flash-Next; do
+            if [[ -d "$candidate" ]]; then
+                SELECTED_MODEL_PATH="$candidate"
+                break
+            fi
+        done
+    fi
+    export MODEL_PATH="${SELECTED_MODEL_PATH:-/scratch/models/Qwen3.8-Flash-Next-NVFP4}"
+    export SRT_SLURM_MODEL_PREFIX="qwen3.8next-fp4"
 else
     echo "Unsupported model prefix/precision: $MODEL_PREFIX/$PRECISION"
     echo "Available models under /scratch/models:"

From 75f3e2c0a04b58923acf80b3a7187cbd9c7fa95e Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Wed, 26 Aug 2026 23:28:48 -0400
Subject: [PATCH 04/12] Use the cookbook's verified TP1 command and the
 committed golden AL
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The recipe was copied from the Qwen3.5 NVFP4 B200 sibling, which is a much
larger model, so it inherited flags this one does not want. The SGLang cookbook
playground's verified single-node command for Qwen3.8-Flash-Next on B200 is
--tp 1: 126 GiB of NVFP4 weights fit on one 192 GB part, so nothing is sharded.
It also takes the hybrid GDN linear-attention backends rather than a single
--attention-backend, reads quantization from the checkpoint instead of
--quantization modelopt_fp4, and drops the trtllm_mha / flashinfer_trtllm MoE
flags that belong to the Qwen3.5 recipe. --max-running-requests stays explicit,
since NEXTN silently resets it to 48 when unset.

Acceptance length moves from the interim 3.24 to 2.32, the thinking_on value at
three speculative tokens from the committed golden curve.

本配方原样照搬自 Qwen3.5 NVFP4 B200 同类配方，而后者模型规模大得多，因此带入了
本模型并不需要的参数。SGLang cookbook playground 给出的 Qwen3.8-Flash-Next
B200 单节点验证命令为 --tp 1：126 GiB 的 NVFP4 权重可放入单张 192 GB 卡，无需
切分。该命令改用混合 GDN 线性注意力的独立后端而非单一 --attention-backend，量化
直接从权重读取而不传 --quantization modelopt_fp4，并去掉了属于 Qwen3.5 配方的
trtllm_mha 与 flashinfer_trtllm MoE 参数。--max-running-requests 保持显式设置，
因为 NEXTN 在其未设置时会静默重置为 48。

接受长度由临时值 3.24 改为 2.32，即已提交黄金曲线中 3 个投机 token 的
thinking_on 取值。

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../qwen3.8next_fp4_b200_sglang_mtp.sh        | 47 ++++++++++---------
 configs/nvidia-master.yaml                    |  7 +--
 perf-changelog.yaml                           |  2 +
 3 files changed, 30 insertions(+), 26 deletions(-)

diff --git a/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh
index 23b1b3fcb1..5a4be31a52 100755
--- a/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh
+++ b/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh
@@ -12,6 +12,10 @@ set -x
 # matches the same flag the Qwen3.5 NVFP4 sibling uses. The model ships native
 # MTP modules (kept unquantized by the checkpoint's ignore list), so NEXTN
 # needs no external drafter.
+#
+# TP1: the cookbook's verified single-node command for this model is --tp 1 on
+# both Blackwell parts. 126 GiB of NVFP4 weights fit on one B200, so the
+# model is not sharded and every rank-crossing collective disappears.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -116,14 +120,12 @@ export SGLANG_ENABLE_FLASHINFER_GEMM=true
 export SGLANG_TIMEOUT_KEEP_ALIVE=1800
 
 if [ "${EVAL_ONLY:-false}" != "true" ]; then
-    # Qwen3.8-Flash-Next acceptance length, SPEED-Bench coding.
-    # --speculative-num-steps 3 with 4 draft tokens is 3 speculative
-    # tokens per verification step, i.e. the MTP=3 cell -> AL 3.24.
-    # Measured thinking=off in speedbench-al run 33031708148; the
-    # thinking=on collection is still in flight, so this is an
-    # interim value and is not yet a committed golden_al_distribution
-    # curve. Refresh once qwen3.8next_mtp.yaml lands.
-    export SGLANG_SIMULATE_ACC_LEN=3.24
+    # golden_al_distribution/qwen3.8next_mtp.yaml:
+    # qwen3.8-flash-next-fp8.thinking_on[3] = 2.32.
+    # --speculative-num-steps 3 with 4 draft tokens is 3 speculative tokens
+    # per verification step, i.e. the MTP=3 cell. AgentX replays run with
+    # thinking on, so the thinking_on row is the right one.
+    export SGLANG_SIMULATE_ACC_LEN=2.32
     export SGLANG_SIMULATE_ACC_METHOD=match-expected
     export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token
 fi
@@ -136,28 +138,27 @@ SGLANG_CMD=(
     --port "$PORT"
     --trust-remote-code
     "${PARALLEL_ARGS[@]}"
-    --enable-symm-mem
-    --quantization modelopt_fp4
-    --fp4-gemm-backend flashinfer_cutlass
-    --kv-cache-dtype fp8_e4m3
+    # Verified flags from the SGLang cookbook playground for this model on
+    # B200 / NVFP4 / single node. Quantization is read from the
+    # checkpoint, so no --quantization flag; the hybrid GDN linear-attention
+    # layers take their own backends rather than --attention-backend.
+    --linear-attn-prefill-backend flashinfer
+    --linear-attn-decode-backend flashinfer
     --mamba-ssm-dtype bfloat16
-    --attention-backend trtllm_mha
-    --moe-runner-backend flashinfer_trtllm
-    --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS"
+    --speculative-algorithm NEXTN
+    --speculative-num-steps 3
+    --speculative-eagle-topk 1
+    --speculative-num-draft-tokens 4
+    --reasoning-parser auto
+    # NEXTN silently resets --max-running-requests to 48 when it is unset, so
+    # this must stay explicit and sized to the AgentX concurrency.
     --max-running-requests "$MAX_RUNNING_REQUESTS"
-    --max-prefill-tokens 16384
-    --chunked-prefill-size 16384
+    --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS"
     --mem-fraction-static 0.80
     --stream-interval 50
     --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL"
     "${TOKENIZER_ARGS[@]}"
     --tokenizer-path "$MODEL"
-    --reasoning-parser qwen3
-    --tool-call-parser qwen3_coder
-    --speculative-algorithm NEXTN
-    --speculative-num-steps 3
-    --speculative-eagle-topk 1
-    --speculative-num-draft-tokens 4
     --enable-metrics
     --enable-cache-report
     "${CACHE_ARGS[@]}"
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index 7ad5c54851..45dfde995b 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -7394,8 +7394,9 @@ qwen3.5-fp4-b200-sglang-agentic-mtp:
 
 
 # Qwen3.8-Flash-Next NVFP4 AgentX on B200 via SGLang with native NEXTN MTP.
-# Day-zero recipe: SGLang is the plan-of-record engine for this model. TP4 at
-# 126 GiB of NVFP4 weights leaves ample HBM for the 256k-capped agentic traces.
+# Day-zero recipe: SGLang is the plan-of-record engine for this model. TP1 per
+# the cookbook's verified single-node command; 126 GiB of NVFP4 weights fit on
+# one B200, so the model is not sharded.
 qwen3.8next-fp4-b200-sglang-agentic-mtp:
   image: lmsysorg/sglang:qwen38flashnext
   model: RadixArk/Qwen3.8-Flash-Next-NVFP4
@@ -7408,7 +7409,7 @@ qwen3.8next-fp4-b200-sglang-agentic-mtp:
     agentic-coding:
     - dram-utilization: 0.8
       search-space:
-      - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] }
+      - { tp: 1, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16] }
 qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp:
   image: lmsysorg/sglang:nightly-dev-20260818-c0b6474b
   model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 5741adb625..ec0ac41f4a 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6586,4 +6586,6 @@
     - "Serve RadixArk/Qwen3.8-Flash-Next-NVFP4 with modelopt_fp4 quantization, the trtllm_mha attention backend, and the flashinfer_trtllm MoE runner, following the Qwen3.5 NVFP4 B200 sibling."
     - "Pin throughput runs to an interim acceptance length of 3.24 measured at three speculative tokens; eval-only runs keep real target verification."
     - "Route the qwen3.8next fp4 prefix to its staged checkpoint in the B200 nscale launcher, which previously rejected the prefix outright."
+    - "Correct the serve flags to the SGLang cookbook's verified single-node command for this model: TP1 rather than TP4, the flashinfer linear-attention prefill and decode backends, bfloat16 Mamba SSM, and checkpoint-derived quantization."
+    - "Set the acceptance length to the committed golden thinking_on value of 2.32 at three speculative tokens, replacing the interim 3.24."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2751

From 7699a1863800717625b0a3d188929ad2dc506271 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Wed, 26 Aug 2026 23:36:05 -0400
Subject: [PATCH 05/12] Fetch the B200 checkpoint from HuggingFace instead of
 the staging tree
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Qwen3.8-Flash-Next NVFP4 is not pre-staged under /scratch/models, and every
other branch in this launcher resolves to that tree, so the previous commit
would have cleared the prefix gate only to fail on the bind mount. Point the
branch at a writable cache directory on the shared /scratch filesystem instead
and let the bench script's own `hf download --local-dir "$MODEL_PATH"` populate
it on the first run.

That needs MODEL to stay a HuggingFace repo id, because `hf download` cannot
take a local path, while this launcher otherwise overwrites MODEL with
MODEL_PATH for every model. KEEP_HF_MODEL_ID gates that overwrite, and the
directory is created up front because srun fails outright when a bind mount
source is missing.

Qwen3.8-Flash-Next NVFP4 并未预置在 /scratch/models 下，而本 launcher 其余分支
都指向该目录树，因此上一个提交虽然过了前缀校验，却会在绑定挂载处失败。改为指向
共享 /scratch 文件系统上的可写缓存目录，由基准脚本自身的
`hf download --local-dir "$MODEL_PATH"` 在首次运行时填充。

这要求 MODEL 保持为 HuggingFace 仓库 id（`hf download` 不接受本地路径），而本
launcher 对其他模型都会把 MODEL 覆盖为 MODEL_PATH。KEEP_HF_MODEL_ID 用于跳过该
覆盖；目录提前创建，因为绑定挂载源缺失时 srun 会直接失败。

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 perf-changelog.yaml                  |  2 +-
 runners/launch_b200-nscale-compat.sh | 38 +++++++++++++++-------------
 2 files changed, 21 insertions(+), 19 deletions(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index ec0ac41f4a..40223ac240 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6585,7 +6585,7 @@
     - "Add the day-zero Qwen3.8-Flash-Next NVFP4 AgentX recipe on B200 with SGLang native NEXTN MTP at TP4 and concurrency 1/4/8/12/16."
     - "Serve RadixArk/Qwen3.8-Flash-Next-NVFP4 with modelopt_fp4 quantization, the trtllm_mha attention backend, and the flashinfer_trtllm MoE runner, following the Qwen3.5 NVFP4 B200 sibling."
     - "Pin throughput runs to an interim acceptance length of 3.24 measured at three speculative tokens; eval-only runs keep real target verification."
-    - "Route the qwen3.8next fp4 prefix to its staged checkpoint in the B200 nscale launcher, which previously rejected the prefix outright."
+    - "Route the qwen3.8next fp4 prefix in the B200 nscale launcher, which previously rejected the prefix outright, and fetch the checkpoint from HuggingFace into a shared-storage cache instead of requiring it pre-staged under the models tree."
     - "Correct the serve flags to the SGLang cookbook's verified single-node command for this model: TP1 rather than TP4, the flashinfer linear-attention prefill and decode backends, bfloat16 Mamba SSM, and checkpoint-derived quantization."
     - "Set the acceptance length to the committed golden thinking_on value of 2.32 at three speculative tokens, replacing the interim 3.24."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2751
diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh
index 1af088fa0e..a70c8792ec 100644
--- a/runners/launch_b200-nscale-compat.sh
+++ b/runners/launch_b200-nscale-compat.sh
@@ -93,23 +93,15 @@ elif [[ $MODEL_PREFIX == "kimik3" && $PRECISION == "fp4" ]]; then
     export MODEL_PATH="/scratch/models/Kimi-K3"
     export SRT_SLURM_MODEL_PREFIX="kimik3"
 elif [[ $MODEL_PREFIX == "qwen3.8next" && $PRECISION == "fp4" ]]; then
-    # Qwen3.8-Flash-Next NVFP4. Like the dsv4 branch, prefer an explicitly
-    # supplied MODEL_PATH, then a staged directory, so a differently named
-    # staging dir does not need a code change here. This launcher bind-mounts
-    # MODEL_PATH into the container and exports MODEL=$MODEL_PATH, so there is
-    # no hf-download fallback: the checkpoint must be staged on the cluster.
-    SELECTED_MODEL_PATH=""
-    if [[ -n "${MODEL_PATH:-}" && -d "${MODEL_PATH}" ]]; then
-        SELECTED_MODEL_PATH="$MODEL_PATH"
-    else
-        for candidate in /scratch/models/Qwen3.8-Flash-Next-NVFP4 /scratch/models/Qwen3.8-Flash-Next-FP8 /scratch/models/Qwen3.8-Flash-Next; do
-            if [[ -d "$candidate" ]]; then
-                SELECTED_MODEL_PATH="$candidate"
-                break
-            fi
-        done
-    fi
-    export MODEL_PATH="${SELECTED_MODEL_PATH:-/scratch/models/Qwen3.8-Flash-Next-NVFP4}"
+    # Qwen3.8-Flash-Next NVFP4 is not pre-staged under /scratch/models, so this
+    # branch does not point at the staging tree. It hands the bench script a
+    # writable cache directory on the shared /scratch filesystem and lets the
+    # script's own `hf download --local-dir "$MODEL_PATH"` populate it on the
+    # first run; later runs find it already there. KEEP_HF_MODEL_ID keeps MODEL
+    # as the HuggingFace repo id further down, because `hf download` needs a
+    # repo id and every other branch here overwrites MODEL with the local path.
+    export MODEL_PATH="${MODEL_PATH:-/scratch/hf-models/Qwen3.8-Flash-Next-NVFP4}"
+    export KEEP_HF_MODEL_ID=1
     export SRT_SLURM_MODEL_PREFIX="qwen3.8next-fp4"
 else
     echo "Unsupported model prefix/precision: $MODEL_PREFIX/$PRECISION"
@@ -595,7 +587,17 @@ else
     # Point the bench script at the resolved MODEL_PATH instead of
     # pulling from the HF hub cache. Bench scripts skip `hf download` when
     # MODEL is a local path.
-    export MODEL="$MODEL_PATH"
+    if [[ "${KEEP_HF_MODEL_ID:-0}" == "1" ]]; then
+        # The bench script downloads into MODEL_PATH itself, so MODEL has to
+        # stay a HuggingFace repo id. Create the directory first: it is bind
+        # mounted below and srun fails outright on a missing mount source.
+        if ! mkdir -p "$MODEL_PATH"; then
+            echo "Error: cannot create $MODEL_PATH on shared storage; the bind mount below would fail" >&2
+            exit 1
+        fi
+    else
+        export MODEL="$MODEL_PATH"
+    fi
 
     # Use flock to serialize concurrent imports to the same squash file
     # Override ENROOT_CACHE_PATH to avoid permission issues with system-wide cache on worker nodes

From 1dd9091243d23cb2c03b5ea2a86c5ad6d69e1b51 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Wed, 26 Aug 2026 23:40:48 -0400
Subject: [PATCH 06/12] Put the B200 checkpoint on the one filesystem both
 sides share
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Verified directly on the nscale cluster: /scratch is not the same filesystem on
both sides of a job. On the login node it is a symlink to NFS /data/scratch,
which is root-owned and holds almost nothing. On a compute node it is
node-local /dev/md0 xfs, which holds the staged models and is not writable by
the runner account. So the previous commit's /scratch/hf-models path would have
been created on NFS by the launcher and then not existed on the compute node
where srun evaluates the bind mount.

The home directory is the one path that is shared, writable and identical on
both: /data/home/sa-shared on 24T of NFS with 5.8T free, confirmed writable
from a compute node. Point MODEL_PATH there. Qwen3.8-Flash-Next is staged
nowhere on this cluster, so the first run downloads it.

已在 nscale 集群上直接核实：/scratch 在作业两侧并非同一个文件系统。登录节点上它是
指向 NFS /data/scratch 的符号链接，属 root 且几乎为空；计算节点上它是节点本地的
/dev/md0 xfs，存放已预置的权重，且运行账号不可写。因此上一个提交使用的
/scratch/hf-models 路径会由 launcher 在 NFS 上创建，而在 srun 评估绑定挂载的计算
节点上并不存在。

家目录是两侧共享、可写且路径一致的唯一位置：/data/home/sa-shared，位于 24T NFS
上，剩余 5.8T，已确认可从计算节点写入。MODEL_PATH 改指该处。该集群未预置
Qwen3.8-Flash-Next，首次运行将自行下载。

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 runners/launch_b200-nscale-compat.sh | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh
index a70c8792ec..45b6a24e58 100644
--- a/runners/launch_b200-nscale-compat.sh
+++ b/runners/launch_b200-nscale-compat.sh
@@ -93,14 +93,18 @@ elif [[ $MODEL_PREFIX == "kimik3" && $PRECISION == "fp4" ]]; then
     export MODEL_PATH="/scratch/models/Kimi-K3"
     export SRT_SLURM_MODEL_PREFIX="kimik3"
 elif [[ $MODEL_PREFIX == "qwen3.8next" && $PRECISION == "fp4" ]]; then
-    # Qwen3.8-Flash-Next NVFP4 is not pre-staged under /scratch/models, so this
-    # branch does not point at the staging tree. It hands the bench script a
-    # writable cache directory on the shared /scratch filesystem and lets the
-    # script's own `hf download --local-dir "$MODEL_PATH"` populate it on the
-    # first run; later runs find it already there. KEEP_HF_MODEL_ID keeps MODEL
-    # as the HuggingFace repo id further down, because `hf download` needs a
-    # repo id and every other branch here overwrites MODEL with the local path.
-    export MODEL_PATH="${MODEL_PATH:-/scratch/hf-models/Qwen3.8-Flash-Next-NVFP4}"
+    # Qwen3.8-Flash-Next NVFP4 is not staged on this cluster, so this branch
+    # does not point at the staging tree. Note /scratch is NOT the same
+    # filesystem on both sides: on the login node it is a symlink to NFS
+    # /data/scratch, while on a compute node it is node-local /dev/md0 xfs that
+    # holds the staged models and is not writable by the runner account. The
+    # home directory is the one path that is shared, writable and identical on
+    # both, so the checkpoint goes there and the bench script's own
+    # `hf download --local-dir "$MODEL_PATH"` populates it on the first run.
+    # KEEP_HF_MODEL_ID keeps MODEL as the HuggingFace repo id further down,
+    # because `hf download` needs a repo id and every other branch here
+    # overwrites MODEL with the local path.
+    export MODEL_PATH="${MODEL_PATH:-${HOME}/models/Qwen3.8-Flash-Next-NVFP4}"
     export KEEP_HF_MODEL_ID=1
     export SRT_SLURM_MODEL_PREFIX="qwen3.8next-fp4"
 else
@@ -593,6 +597,7 @@ else
         # mounted below and srun fails outright on a missing mount source.
         if ! mkdir -p "$MODEL_PATH"; then
             echo "Error: cannot create $MODEL_PATH on shared storage; the bind mount below would fail" >&2
+            echo "Note: compute-node /scratch is node-local and not writable here; use a path under \$HOME" >&2
             exit 1
         fi
     else

From 6b8a276d3c6b3b8f190ea98455c789b1fe34dc5d Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Thu, 27 Aug 2026 00:08:50 -0400
Subject: [PATCH 07/12] Use a float32 Mamba SSM state so the NEXTN verify
 kernel accepts it
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

flashinfer's gated_delta_rule_mtp asserts initial_state.dtype == torch.float32
and aborts CUDA graph capture on the bfloat16 SSM state the cookbook command
specifies. Confirmed on the H200 arm, which died at "Capture target verify
CUDA graph"; this arm runs the same GDN backend, the same kernel and the same
NEXTN, so it would fail identically once it got that far.

flashinfer 的 gated_delta_rule_mtp 断言 initial_state 必须为 float32，遇到
cookbook 命令指定的 bfloat16 SSM 状态会在 CUDA graph 捕获阶段中止。该问题已在
H200 分支确认；本分支使用相同的 GDN 后端、相同内核与相同的 NEXTN，运行到同一阶段
必然同样失败。

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../agentic/qwen3.8next_fp4_b200_sglang_mtp.sh           | 9 ++++++++-
 perf-changelog.yaml                                      | 1 +
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh
index 5a4be31a52..08bd95c5a1 100755
--- a/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh
+++ b/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh
@@ -144,7 +144,14 @@ SGLANG_CMD=(
     # layers take their own backends rather than --attention-backend.
     --linear-attn-prefill-backend flashinfer
     --linear-attn-decode-backend flashinfer
-    --mamba-ssm-dtype bfloat16
+    # float32, not the cookbook's bfloat16. With NEXTN enabled the GDN linear
+    # attention backend routes verification through flashinfer's
+    # gated_delta_rule_mtp, which asserts initial_state.dtype == torch.float32
+    # and aborts CUDA graph capture on a bf16 SSM state:
+    #   AssertionError: initial_state must be float32, got torch.bfloat16
+    #   flashinfer/gdn_decode.py:761, via gdn_backend.py target_verify
+    # Confirmed on the H200 arm; same backend, same kernel, same NEXTN here.
+    --mamba-ssm-dtype float32
     --speculative-algorithm NEXTN
     --speculative-num-steps 3
     --speculative-eagle-topk 1
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 40223ac240..695b7cdf0d 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6588,4 +6588,5 @@
     - "Route the qwen3.8next fp4 prefix in the B200 nscale launcher, which previously rejected the prefix outright, and fetch the checkpoint from HuggingFace into a shared-storage cache instead of requiring it pre-staged under the models tree."
     - "Correct the serve flags to the SGLang cookbook's verified single-node command for this model: TP1 rather than TP4, the flashinfer linear-attention prefill and decode backends, bfloat16 Mamba SSM, and checkpoint-derived quantization."
     - "Set the acceptance length to the committed golden thinking_on value of 2.32 at three speculative tokens, replacing the interim 3.24."
+    - "Use a float32 Mamba SSM state: flashinfer's gated_delta_rule_mtp verify kernel asserts float32 and aborts CUDA graph capture on the bfloat16 state the cookbook command specifies."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2751

From 99994a14f5db02b6bcb9ab1fab7d0c6a6a6ceb11 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Thu, 27 Aug 2026 00:21:34 -0400
Subject: [PATCH 08/12] Keep the bfloat16 SSM state on Blackwell
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

My previous commit copied the H200 float32 fix here, and SGLang refused to
start at all:

  ValueError: --linear-attn-decode-backend flashinfer on SM100+ requires
  --mamba-ssm-dtype bfloat16, got 'float32'

The two architectures want opposite things. On SM100+ the flashinfer linear
attention decode backend requires a bfloat16 state and validates it in
server_args before launch. On Hopper the same backend's MTP verify kernel
asserts a float32 state during CUDA graph capture. So the cookbook's bfloat16
is correct here and the H200 arm is the exception, not the other way round.
Reverted, with the asymmetry written down so neither arm gets 'fixed' to match
the other again.

上一个提交把 H200 的 float32 改动照搬到本分支，导致 SGLang 直接拒绝启动：SM100+
上使用 flashinfer 线性注意力解码后端时必须搭配 bfloat16。两种架构的要求正好相反：
SM100+ 在 server_args 中校验并要求 bfloat16；而 Hopper 上同一后端的 MTP 验证内核
在 CUDA graph 捕获阶段要求 float32。因此 cookbook 的 bfloat16 在此处是正确的，
H200 才是例外。现已回退，并把这一不对称写入注释，避免两侧再被互相“对齐”。

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../agentic/qwen3.8next_fp4_b200_sglang_mtp.sh     | 14 ++++++--------
 perf-changelog.yaml                                |  2 +-
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh
index 08bd95c5a1..de36b5964f 100755
--- a/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh
+++ b/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh
@@ -144,14 +144,12 @@ SGLANG_CMD=(
     # layers take their own backends rather than --attention-backend.
     --linear-attn-prefill-backend flashinfer
     --linear-attn-decode-backend flashinfer
-    # float32, not the cookbook's bfloat16. With NEXTN enabled the GDN linear
-    # attention backend routes verification through flashinfer's
-    # gated_delta_rule_mtp, which asserts initial_state.dtype == torch.float32
-    # and aborts CUDA graph capture on a bf16 SSM state:
-    #   AssertionError: initial_state must be float32, got torch.bfloat16
-    #   flashinfer/gdn_decode.py:761, via gdn_backend.py target_verify
-    # Confirmed on the H200 arm; same backend, same kernel, same NEXTN here.
-    --mamba-ssm-dtype float32
+    # bfloat16 is mandatory on Blackwell: SGLang rejects the launch outright
+    # with "--linear-attn-decode-backend flashinfer on SM100+ requires
+    # --mamba-ssm-dtype bfloat16". Hopper wants the opposite -- flashinfer's
+    # gated_delta_rule_mtp verify kernel asserts a float32 state there -- so
+    # the H200 arm sets float32 and this one must not follow it.
+    --mamba-ssm-dtype bfloat16
     --speculative-algorithm NEXTN
     --speculative-num-steps 3
     --speculative-eagle-topk 1
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 695b7cdf0d..767fe08f32 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6588,5 +6588,5 @@
     - "Route the qwen3.8next fp4 prefix in the B200 nscale launcher, which previously rejected the prefix outright, and fetch the checkpoint from HuggingFace into a shared-storage cache instead of requiring it pre-staged under the models tree."
     - "Correct the serve flags to the SGLang cookbook's verified single-node command for this model: TP1 rather than TP4, the flashinfer linear-attention prefill and decode backends, bfloat16 Mamba SSM, and checkpoint-derived quantization."
     - "Set the acceptance length to the committed golden thinking_on value of 2.32 at three speculative tokens, replacing the interim 3.24."
-    - "Use a float32 Mamba SSM state: flashinfer's gated_delta_rule_mtp verify kernel asserts float32 and aborts CUDA graph capture on the bfloat16 state the cookbook command specifies."
+    - "Keep the bfloat16 Mamba SSM state the cookbook specifies: SGLang requires it on SM100 or newer whenever the flashinfer linear-attention decode backend is selected."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2751

From b5f113d0acd3c6769be2d59d9314d0f634ec163e Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Thu, 27 Aug 2026 01:52:48 -0400
Subject: [PATCH 09/12] Only accept a complete checkpoint, and serialize the
 download
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

B300 passed its evals; B200 ran the identical recipe and returned
gsm8k exact_match 0.0000 on both strict-match and flexible-extract. The server
started and served, so this was not an infrastructure failure: it was serving a
broken model.

Inspecting the nscale cluster showed why. The shared checkpoint directory held
81 GB of 126 GB, 396 of 419 files, 8 leftover .incomplete files and no
model.safetensors.index.json. The download had stalled, and the guard here only
asked whether the directory was non-empty, so every cell accepted the partial
tree and loaded whatever shards happened to be present. The eval gate is what
caught it; the throughput numbers from those cells would otherwise have been
published as if they were real.

Three changes, all inside the script rather than by pre-staging weights:

  * Completeness is now decided by the index and every shard it names, plus the
    absence of .incomplete files, instead of by emptiness. hf download is
    incremental, so re-running it against a partial tree resumes cheaply.
  * The download takes a flock around the shared directory. Checking
    completeness alone would have turned "one cell downloads" into "all six
    download into the same tree at once", and hf download names .incomplete
    files by content hash, so concurrent writers collide on the same paths.
    The first holder downloads; the rest wake, find it complete and skip. The
    launcher already uses flock this way for squashfs imports.
  * HF_HUB_DISABLE_XET=1 for the download. Xet stalled this transfer twice at
    exactly 81 GB, once from a login node and once from inside a job; the plain
    HTTPS path sustained 861 MB/s.

The stale 81 GB tree has been removed from the cluster so the next run starts
clean. Also verified the completeness helper against an empty directory, an
index with no shards, a partial shard set, a full set, and a leftover
.incomplete file.

B300 的评测通过，而 B200 以完全相同的配方返回 gsm8k exact_match 0.0000。服务本身
正常启动并响应，因此这不是基础设施故障，而是在为一个损坏的模型提供服务。

在 nscale 集群上核查发现：共享权重目录只有 126 GB 中的 81 GB、419 个文件中的 396
个、8 个残留 .incomplete 文件，且缺少 model.safetensors.index.json。下载中途停滞，
而此处的判据仅检查目录是否非空，于是每个单元都接受了这棵残缺的目录树并加载了碰巧
存在的分片。是评测环节兜住了这个问题，否则这些单元的吞吐数据会被当作有效结果发布。

三处改动，全部在脚本内完成，不采用预置权重的做法：完整性改为依据索引及其列出的每个
分片、并要求不存在 .incomplete 文件；下载前对共享目录加 flock（仅做完整性检查会使
「一个单元下载」变成「六个单元同时下载同一棵目录树」，而 hf download 的 .incomplete
文件按内容哈希命名，并发写入会相互冲突）；下载时设置 HF_HUB_DISABLE_XET=1（Xet 两次
都恰好停滞在 81 GB，而普通 HTTPS 路径可达 861 MB/s）。

集群上残留的 81 GB 目录已删除，下次运行将从干净状态开始。完整性判据也已针对空目录、
仅有索引、分片不全、分片齐全以及残留 .incomplete 五种情况验证。

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../qwen3.8next_fp4_b200_sglang_mtp.sh        | 75 ++++++++++++++++++-
 perf-changelog.yaml                           |  1 +
 2 files changed, 73 insertions(+), 3 deletions(-)

diff --git a/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh
index de36b5964f..8fb8c74ecc 100755
--- a/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh
+++ b/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh
@@ -32,10 +32,79 @@ if [[ -n "${SLURM_JOB_ID:-}" ]]; then
     echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
 fi
 
-if [[ -n "${MODEL_PATH:-}" ]]; then
-    if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
-        hf download "$MODEL" --local-dir "$MODEL_PATH"
+# The B200 nscale launcher hands us a shared directory rather than a staged
+# checkpoint, and several matrix cells run against it at once, so "non-empty"
+# is not the same as "complete". A first cell whose download stalls leaves a
+# partial tree that every later cell accepts and serves: that is how this arm
+# produced a fully loaded server and gsm8k exact_match 0.0000, with 81 GB of
+# 126 GB on disk, 8 leftover .incomplete files and no index. Check the index
+# and every shard it names instead, and let hf download resume the rest --
+# it is incremental, so re-running it on a partial tree is cheap and correct.
+model_download_is_complete() {
+    local dir="$1"
+    local index="$dir/model.safetensors.index.json"
+    [[ -f "$index" ]] || return 1
+    if [[ -n "$(find "$dir" -name '*.incomplete' -print -quit 2>/dev/null)" ]]; then
+        return 1
+    fi
+    python3 - "$dir" <<'PYEOF'
+import json, os, sys
+
+directory = sys.argv[1]
+index = os.path.join(directory, "model.safetensors.index.json")
+try:
+    shards = set(json.load(open(index))["weight_map"].values())
+except Exception:
+    sys.exit(1)
+missing = [s for s in sorted(shards) if not os.path.isfile(os.path.join(directory, s))]
+if missing:
+    print(f"checkpoint incomplete: {len(missing)}/{len(shards)} shards missing, "
+          f"first {missing[0]}", file=sys.stderr)
+    sys.exit(1)
+sys.exit(0)
+PYEOF
+}
+
+download_model_once() {
+    # Serialize across the matrix cells sharing this directory. Without the
+    # lock the completeness check turns "one cell downloads" into "every cell
+    # downloads into the same tree at once", and hf download names its
+    # .incomplete files by content hash, so the concurrent writers collide on
+    # the same paths. The first holder downloads; the rest wake up, find the
+    # checkpoint complete and skip. flock matches what the launcher already
+    # does for squashfs imports.
+    local lock="${MODEL_PATH}.lock"
+    mkdir -p "$(dirname "$MODEL_PATH")"
+    if ! command -v flock >/dev/null 2>&1; then
+        echo "WARNING: flock unavailable; downloading without the cross-cell lock" >&2
+        _download_model_if_needed
+        return
+    fi
+    exec 9>"$lock"
+    if ! flock -w 5400 9; then
+        echo "WARNING: timed out waiting for $lock; proceeding unlocked" >&2
+    fi
+    _download_model_if_needed
+    exec 9>&-
+}
+
+_download_model_if_needed() {
+    if model_download_is_complete "$MODEL_PATH"; then
+        echo "Checkpoint already complete at $MODEL_PATH"
+        return
+    fi
+    echo "Downloading $MODEL into $MODEL_PATH (absent or incomplete)"
+    # Xet stalled this transfer twice at exactly 81 GB, once from a login node
+    # and once from inside a job. The plain HTTPS path does not.
+    HF_HUB_DISABLE_XET=1 hf download "$MODEL" --local-dir "$MODEL_PATH"
+    if ! model_download_is_complete "$MODEL_PATH"; then
+        echo "Error: $MODEL_PATH is still incomplete after hf download" >&2
+        exit 1
     fi
+}
+
+if [[ -n "${MODEL_PATH:-}" ]]; then
+    download_model_once
 else
     hf download "$MODEL"
     export MODEL_PATH="$MODEL"
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index 767fe08f32..ade4d0f470 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6589,4 +6589,5 @@
     - "Correct the serve flags to the SGLang cookbook's verified single-node command for this model: TP1 rather than TP4, the flashinfer linear-attention prefill and decode backends, bfloat16 Mamba SSM, and checkpoint-derived quantization."
     - "Set the acceptance length to the committed golden thinking_on value of 2.32 at three speculative tokens, replacing the interim 3.24."
     - "Keep the bfloat16 Mamba SSM state the cookbook specifies: SGLang requires it on SM100 or newer whenever the flashinfer linear-attention decode backend is selected."
+    - "Treat the shared checkpoint directory as complete only when the index and every shard it names are present, take a cross-cell lock around the download, and disable Xet for it, after a stalled partial download was served as a finished model."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2751

From eb44fa048addb8ea05bb7adec895b7c01baa8ae4 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Thu, 27 Aug 2026 02:05:40 -0400
Subject: [PATCH 10/12] Download the B200 checkpoint to node-local scratch
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The completeness check and the flock were not enough. The next run failed with

  OSError: [Errno 521] Unknown error 521:
  /data/home/sa-shared/models/Qwen3.8-Flash-Next-NVFP4/layer-00007-experts-0256-0383.safetensors

raised from huggingface_hub reading a shard back, i.e. a stale-handle class
error on NFSv4 while another cell was writing the same tree from a different
node.

The shared home was the wrong target. Every cell in the matrix runs on its own
node and they were all writing one directory over NFS. /tmp on these compute
nodes is /dev/md0 with about 17 TB free, so point MODEL_PATH there: each node
fetches its own copy, roughly 2.5 minutes at the 861 MB/s these nodes sustain,
and cross-node sharing disappears along with the lock semantics it depended on.

The directory is now created with srun so it lands on the allocated node; the
login node has an unrelated /tmp, and the bind mount source has to exist where
the container starts. The completeness check stays: it still guards a partial
tree within a node, and it is what turned this from a silent gsm8k 0.0000 into
a visible failure.

完整性检查与 flock 仍不足够。下一次运行报错：huggingface_hub 在回读分片时抛出
OSError: [Errno 521]，即 NFSv4 上的失效句柄类错误——彼时另一个单元正从另一个节点
写入同一棵目录树。

共享家目录本就是错误的落点：矩阵中每个单元各自占用一个节点，却都在通过 NFS 写同一
个目录。这些计算节点的 /tmp 位于 /dev/md0，可用约 17 TB，因此将 MODEL_PATH 指向
该处：每个节点各自下载一份（按这些节点实测的 861 MB/s 约需 2.5 分钟），跨节点共享
及其依赖的锁语义一并消失。

目录改用 srun 创建，以确保建在已分配的计算节点上——登录节点的 /tmp 与之无关，而绑定
挂载的源必须存在于容器启动的那台机器上。完整性检查予以保留：它仍能防止单节点内的
残缺目录树，也正是它把一次静默的 gsm8k 0.0000 变成了可见的失败。

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 perf-changelog.yaml                  |  2 +-
 runners/launch_b200-nscale-compat.sh | 34 ++++++++++++++++------------
 2 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index ade4d0f470..e2bb81848a 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6589,5 +6589,5 @@
     - "Correct the serve flags to the SGLang cookbook's verified single-node command for this model: TP1 rather than TP4, the flashinfer linear-attention prefill and decode backends, bfloat16 Mamba SSM, and checkpoint-derived quantization."
     - "Set the acceptance length to the committed golden thinking_on value of 2.32 at three speculative tokens, replacing the interim 3.24."
     - "Keep the bfloat16 Mamba SSM state the cookbook specifies: SGLang requires it on SM100 or newer whenever the flashinfer linear-attention decode backend is selected."
-    - "Treat the shared checkpoint directory as complete only when the index and every shard it names are present, take a cross-cell lock around the download, and disable Xet for it, after a stalled partial download was served as a finished model."
+    - "Download the checkpoint to node-local scratch rather than the shared home, and treat it as complete only when the index and every shard it names are present, after a stalled partial download was served as a finished model and a shared-home download hit a stale-handle error."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2751
diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh
index 45b6a24e58..12c3bb0ae6 100644
--- a/runners/launch_b200-nscale-compat.sh
+++ b/runners/launch_b200-nscale-compat.sh
@@ -94,17 +94,21 @@ elif [[ $MODEL_PREFIX == "kimik3" && $PRECISION == "fp4" ]]; then
     export SRT_SLURM_MODEL_PREFIX="kimik3"
 elif [[ $MODEL_PREFIX == "qwen3.8next" && $PRECISION == "fp4" ]]; then
     # Qwen3.8-Flash-Next NVFP4 is not staged on this cluster, so this branch
-    # does not point at the staging tree. Note /scratch is NOT the same
-    # filesystem on both sides: on the login node it is a symlink to NFS
-    # /data/scratch, while on a compute node it is node-local /dev/md0 xfs that
-    # holds the staged models and is not writable by the runner account. The
-    # home directory is the one path that is shared, writable and identical on
-    # both, so the checkpoint goes there and the bench script's own
-    # `hf download --local-dir "$MODEL_PATH"` populates it on the first run.
-    # KEEP_HF_MODEL_ID keeps MODEL as the HuggingFace repo id further down,
-    # because `hf download` needs a repo id and every other branch here
-    # overwrites MODEL with the local path.
-    export MODEL_PATH="${MODEL_PATH:-${HOME}/models/Qwen3.8-Flash-Next-NVFP4}"
+    # does not point at the staging tree. The bench script's own
+    # `hf download --local-dir "$MODEL_PATH"` populates it instead, and
+    # KEEP_HF_MODEL_ID below keeps MODEL a HuggingFace repo id so that call has
+    # something valid to fetch.
+    #
+    # The target is node-local scratch, not the shared home. /tmp on a compute
+    # node is /dev/md0 with ~17 TB free; $HOME is NFSv4. Downloading 126 GiB to
+    # the shared home made every cell in the matrix write the same tree from a
+    # different node, and huggingface_hub failed reading a shard back with
+    #   OSError: [Errno 521] ... layer-00007-experts-0256-0383.safetensors
+    # a stale-handle class error. Node-local storage makes each node fetch its
+    # own copy -- about 2.5 minutes at the 861 MB/s these nodes sustain -- and
+    # removes cross-node sharing from the picture entirely. The directory is
+    # created with srun so it lands on the allocated node.
+    export MODEL_PATH="${MODEL_PATH:-/tmp/inferencex-models/Qwen3.8-Flash-Next-NVFP4}"
     export KEEP_HF_MODEL_ID=1
     export SRT_SLURM_MODEL_PREFIX="qwen3.8next-fp4"
 else
@@ -595,9 +599,11 @@ else
         # The bench script downloads into MODEL_PATH itself, so MODEL has to
         # stay a HuggingFace repo id. Create the directory first: it is bind
         # mounted below and srun fails outright on a missing mount source.
-        if ! mkdir -p "$MODEL_PATH"; then
-            echo "Error: cannot create $MODEL_PATH on shared storage; the bind mount below would fail" >&2
-            echo "Note: compute-node /scratch is node-local and not writable here; use a path under \$HOME" >&2
+        # Create it ON THE ALLOCATED NODE, because MODEL_PATH is node-local
+        # scratch: /tmp is /dev/md0 with ~17 TB free on these nodes, while the
+        # login node has an unrelated /tmp of its own.
+        if ! srun --jobid="$JOB_ID" mkdir -p "$MODEL_PATH"; then
+            echo "Error: cannot create $MODEL_PATH on the compute node; the bind mount below would fail" >&2
             exit 1
         fi
     else

From 5462307065a07b6d40066d90ea76d56875ea6b2f Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Thu, 27 Aug 2026 02:10:02 -0400
Subject: [PATCH 11/12] Use the staged checkpoint now that it is on the cluster
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Qwen3.8-Flash-Next NVFP4 is now staged at /scratch/models on the compute nodes,
verified on im-b200-c004: 126 GB, index present, 206/206 shards, no .incomplete
leftovers. That removes the reason for everything this arm accumulated while
the weights were absent.

Deleted: the hf-download fallback and KEEP_HF_MODEL_ID in the launcher, the
srun mkdir of a node-local target, and the bench script's completeness check,
cross-cell flock and HF_HUB_DISABLE_XET. The launcher branch is now the
ordinary staged-path lookup with a dsv4-style candidate search, and the bench
script is byte-identical to the merged B300 sibling apart from the SKU name.

Every one of those existed only to work around a missing checkpoint: the
partial 81 GB tree that was served as a finished model and produced gsm8k
0.0000, and the NFSv4 stale-handle error from six nodes writing one shared
directory. With the checkpoint staged, none of it applies, and B300 already
passes with exactly this recipe.

Qwen3.8-Flash-Next NVFP4 现已预置在计算节点的 /scratch/models 下，已在
im-b200-c004 上核实：126 GB、索引齐全、206/206 分片、无 .incomplete 残留。此前为
应对权重缺失而在本分支上累积的各项处理，至此均无必要。

已删除：launcher 中的 hf 下载兜底与 KEEP_HF_MODEL_ID、在节点本地创建目标目录的
srun mkdir，以及基准脚本中的完整性检查、跨单元 flock 与 HF_HUB_DISABLE_XET。
launcher 分支恢复为常规的预置路径查找（沿用 dsv4 式的候选目录搜索），基准脚本除
SKU 名称外与已合入的 B300 同类脚本逐字节一致。

这些处理本就只为绕开权重缺失：被当作完整模型加载、导致 gsm8k 0.0000 的 81 GB 残缺
目录树，以及六个节点同写一个共享目录引发的 NFSv4 失效句柄错误。权重预置后均不再适
用，而 B300 正是以这套配方通过的。

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../qwen3.8next_fp4_b200_sglang_mtp.sh        | 75 +------------------
 perf-changelog.yaml                           |  3 +-
 runners/launch_b200-nscale-compat.sh          | 49 +++++-------
 3 files changed, 22 insertions(+), 105 deletions(-)

diff --git a/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh
index 8fb8c74ecc..de36b5964f 100755
--- a/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh
+++ b/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh
@@ -32,79 +32,10 @@ if [[ -n "${SLURM_JOB_ID:-}" ]]; then
     echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
 fi
 
-# The B200 nscale launcher hands us a shared directory rather than a staged
-# checkpoint, and several matrix cells run against it at once, so "non-empty"
-# is not the same as "complete". A first cell whose download stalls leaves a
-# partial tree that every later cell accepts and serves: that is how this arm
-# produced a fully loaded server and gsm8k exact_match 0.0000, with 81 GB of
-# 126 GB on disk, 8 leftover .incomplete files and no index. Check the index
-# and every shard it names instead, and let hf download resume the rest --
-# it is incremental, so re-running it on a partial tree is cheap and correct.
-model_download_is_complete() {
-    local dir="$1"
-    local index="$dir/model.safetensors.index.json"
-    [[ -f "$index" ]] || return 1
-    if [[ -n "$(find "$dir" -name '*.incomplete' -print -quit 2>/dev/null)" ]]; then
-        return 1
-    fi
-    python3 - "$dir" <<'PYEOF'
-import json, os, sys
-
-directory = sys.argv[1]
-index = os.path.join(directory, "model.safetensors.index.json")
-try:
-    shards = set(json.load(open(index))["weight_map"].values())
-except Exception:
-    sys.exit(1)
-missing = [s for s in sorted(shards) if not os.path.isfile(os.path.join(directory, s))]
-if missing:
-    print(f"checkpoint incomplete: {len(missing)}/{len(shards)} shards missing, "
-          f"first {missing[0]}", file=sys.stderr)
-    sys.exit(1)
-sys.exit(0)
-PYEOF
-}
-
-download_model_once() {
-    # Serialize across the matrix cells sharing this directory. Without the
-    # lock the completeness check turns "one cell downloads" into "every cell
-    # downloads into the same tree at once", and hf download names its
-    # .incomplete files by content hash, so the concurrent writers collide on
-    # the same paths. The first holder downloads; the rest wake up, find the
-    # checkpoint complete and skip. flock matches what the launcher already
-    # does for squashfs imports.
-    local lock="${MODEL_PATH}.lock"
-    mkdir -p "$(dirname "$MODEL_PATH")"
-    if ! command -v flock >/dev/null 2>&1; then
-        echo "WARNING: flock unavailable; downloading without the cross-cell lock" >&2
-        _download_model_if_needed
-        return
-    fi
-    exec 9>"$lock"
-    if ! flock -w 5400 9; then
-        echo "WARNING: timed out waiting for $lock; proceeding unlocked" >&2
-    fi
-    _download_model_if_needed
-    exec 9>&-
-}
-
-_download_model_if_needed() {
-    if model_download_is_complete "$MODEL_PATH"; then
-        echo "Checkpoint already complete at $MODEL_PATH"
-        return
-    fi
-    echo "Downloading $MODEL into $MODEL_PATH (absent or incomplete)"
-    # Xet stalled this transfer twice at exactly 81 GB, once from a login node
-    # and once from inside a job. The plain HTTPS path does not.
-    HF_HUB_DISABLE_XET=1 hf download "$MODEL" --local-dir "$MODEL_PATH"
-    if ! model_download_is_complete "$MODEL_PATH"; then
-        echo "Error: $MODEL_PATH is still incomplete after hf download" >&2
-        exit 1
-    fi
-}
-
 if [[ -n "${MODEL_PATH:-}" ]]; then
-    download_model_once
+    if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
+        hf download "$MODEL" --local-dir "$MODEL_PATH"
+    fi
 else
     hf download "$MODEL"
     export MODEL_PATH="$MODEL"
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
index e2bb81848a..f82d613bf6 100644
--- a/perf-changelog.yaml
+++ b/perf-changelog.yaml
@@ -6585,9 +6585,8 @@
     - "Add the day-zero Qwen3.8-Flash-Next NVFP4 AgentX recipe on B200 with SGLang native NEXTN MTP at TP4 and concurrency 1/4/8/12/16."
     - "Serve RadixArk/Qwen3.8-Flash-Next-NVFP4 with modelopt_fp4 quantization, the trtllm_mha attention backend, and the flashinfer_trtllm MoE runner, following the Qwen3.5 NVFP4 B200 sibling."
     - "Pin throughput runs to an interim acceptance length of 3.24 measured at three speculative tokens; eval-only runs keep real target verification."
-    - "Route the qwen3.8next fp4 prefix in the B200 nscale launcher, which previously rejected the prefix outright, and fetch the checkpoint from HuggingFace into a shared-storage cache instead of requiring it pre-staged under the models tree."
     - "Correct the serve flags to the SGLang cookbook's verified single-node command for this model: TP1 rather than TP4, the flashinfer linear-attention prefill and decode backends, bfloat16 Mamba SSM, and checkpoint-derived quantization."
     - "Set the acceptance length to the committed golden thinking_on value of 2.32 at three speculative tokens, replacing the interim 3.24."
     - "Keep the bfloat16 Mamba SSM state the cookbook specifies: SGLang requires it on SM100 or newer whenever the flashinfer linear-attention decode backend is selected."
-    - "Download the checkpoint to node-local scratch rather than the shared home, and treat it as complete only when the index and every shard it names are present, after a stalled partial download was served as a finished model and a shared-home download hit a stale-handle error."
+    - "Route the qwen3.8next fp4 prefix in the B200 nscale launcher to its staged checkpoint under the models tree; the launcher previously rejected the prefix outright."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2751
diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh
index 12c3bb0ae6..9dea333021 100644
--- a/runners/launch_b200-nscale-compat.sh
+++ b/runners/launch_b200-nscale-compat.sh
@@ -93,23 +93,23 @@ elif [[ $MODEL_PREFIX == "kimik3" && $PRECISION == "fp4" ]]; then
     export MODEL_PATH="/scratch/models/Kimi-K3"
     export SRT_SLURM_MODEL_PREFIX="kimik3"
 elif [[ $MODEL_PREFIX == "qwen3.8next" && $PRECISION == "fp4" ]]; then
-    # Qwen3.8-Flash-Next NVFP4 is not staged on this cluster, so this branch
-    # does not point at the staging tree. The bench script's own
-    # `hf download --local-dir "$MODEL_PATH"` populates it instead, and
-    # KEEP_HF_MODEL_ID below keeps MODEL a HuggingFace repo id so that call has
-    # something valid to fetch.
-    #
-    # The target is node-local scratch, not the shared home. /tmp on a compute
-    # node is /dev/md0 with ~17 TB free; $HOME is NFSv4. Downloading 126 GiB to
-    # the shared home made every cell in the matrix write the same tree from a
-    # different node, and huggingface_hub failed reading a shard back with
-    #   OSError: [Errno 521] ... layer-00007-experts-0256-0383.safetensors
-    # a stale-handle class error. Node-local storage makes each node fetch its
-    # own copy -- about 2.5 minutes at the 861 MB/s these nodes sustain -- and
-    # removes cross-node sharing from the picture entirely. The directory is
-    # created with srun so it lands on the allocated node.
-    export MODEL_PATH="${MODEL_PATH:-/tmp/inferencex-models/Qwen3.8-Flash-Next-NVFP4}"
-    export KEEP_HF_MODEL_ID=1
+    # Staged on the compute nodes like every other model here, so this is the
+    # ordinary branch again: no hf download and no writable target needed.
+    # Verified on im-b200-c004: 126 GB, index present, 206/206 shards, no
+    # .incomplete leftovers. The candidate search mirrors the dsv4 branch so a
+    # differently named staging directory needs no code change.
+    SELECTED_MODEL_PATH=""
+    if [[ -n "${MODEL_PATH:-}" && -d "${MODEL_PATH}" ]]; then
+        SELECTED_MODEL_PATH="$MODEL_PATH"
+    else
+        for candidate in /scratch/models/Qwen3.8-Flash-Next-NVFP4 /scratch/models/Qwen3.8-Flash-Next; do
+            if [[ -d "$candidate" ]]; then
+                SELECTED_MODEL_PATH="$candidate"
+                break
+            fi
+        done
+    fi
+    export MODEL_PATH="${SELECTED_MODEL_PATH:-/scratch/models/Qwen3.8-Flash-Next-NVFP4}"
     export SRT_SLURM_MODEL_PREFIX="qwen3.8next-fp4"
 else
     echo "Unsupported model prefix/precision: $MODEL_PREFIX/$PRECISION"
@@ -595,20 +595,7 @@ else
     # Point the bench script at the resolved MODEL_PATH instead of
     # pulling from the HF hub cache. Bench scripts skip `hf download` when
     # MODEL is a local path.
-    if [[ "${KEEP_HF_MODEL_ID:-0}" == "1" ]]; then
-        # The bench script downloads into MODEL_PATH itself, so MODEL has to
-        # stay a HuggingFace repo id. Create the directory first: it is bind
-        # mounted below and srun fails outright on a missing mount source.
-        # Create it ON THE ALLOCATED NODE, because MODEL_PATH is node-local
-        # scratch: /tmp is /dev/md0 with ~17 TB free on these nodes, while the
-        # login node has an unrelated /tmp of its own.
-        if ! srun --jobid="$JOB_ID" mkdir -p "$MODEL_PATH"; then
-            echo "Error: cannot create $MODEL_PATH on the compute node; the bind mount below would fail" >&2
-            exit 1
-        fi
-    else
-        export MODEL="$MODEL_PATH"
-    fi
+    export MODEL="$MODEL_PATH"
 
     # Use flock to serialize concurrent imports to the same squash file
     # Override ENROOT_CACHE_PATH to avoid permission issues with system-wide cache on worker nodes

From 41be4e331a049b1fa552dd7daa762139479cf8e9 Mon Sep 17 00:00:00 2001
From: functionstackx <47992694+functionstackx@users.noreply.github.com>
Date: Thu, 27 Aug 2026 03:29:57 -0400
Subject: [PATCH 12/12] Re-run the B200 sweep
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The previous run failed a single cell, concurrency 1, on the AIPerf coverage
gate: TTFT 93.7% and inter-token latency 94.0% against a 95% requirement, with
no latency signal in the final 180s of the 3600s window. The other four cells
were cancelled by fail-fast rather than failing on their own, and the eval
passed. Server startup, the staged checkpoint and the serve flags are all
working, so this re-runs to see whether the quiet tail is deterministic.

上一次运行仅有并发 1 这一个单元失败，卡在 AIPerf 覆盖率门限：TTFT 93.7%、
token 间延迟 94.0%，均低于 95% 的要求，且在 3600 秒窗口的最后 180 秒内没有任何
延迟信号。其余四个单元是被 fail-fast 取消而非自身失败，评测单元通过。服务启动、
预置权重与服务参数均已正常，因此重跑一次以确认末段静默是否可复现。

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>