diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 2195bad7e2..053c19a860 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -430,7 +430,7 @@ qwen3.5-fp8-mi300x-sglang-agentic-mtp: image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi30x-20260730 model: Qwen/Qwen3.5-397B-A17B-FP8 model-prefix: qwen3.5 - runner: cluster:mi300x-amds + runner: cluster:mi300x-amd precision: fp8 framework: sglang multinode: false @@ -1516,7 +1516,7 @@ minimaxm3-fp8-mi300x-vllm-agentic-mtp: image: vllm/vllm-openai-rocm:v0.27.1 model: MiniMaxAI/MiniMax-M3-MXFP8 model-prefix: minimaxm3 - runner: cluster:mi300x-amds + runner: cluster:mi300x-amd precision: fp8 framework: vllm multinode: false diff --git a/configs/runners.yaml b/configs/runners.yaml index 6b6fdd0c71..490ff87cc8 100644 --- a/configs/runners.yaml +++ b/configs/runners.yaml @@ -101,19 +101,15 @@ labels: - b200-nscale-slurm_7 - b200-nscale-slurm_8 mi300x: - - mi300x-amds_00 - - mi300x-amds_01 - - mi300x-amds_02 - - mi300x-amds_03 - - mi300x-amds_04 - - mi300x-amds_05 - - mi300x-amds_06 - - mi300x-amds_07 - - mi300x-amds_08 - mi300x-disagg: - - mi300x-amds_06 - - mi300x-amds_07 - - mi300x-amds_08 + - mi300x-amd_00 + - mi300x-amd_01 + - mi300x-amd_02 + - mi300x-amd_03 + - mi300x-amd_04 + - mi300x-amd_05 + - mi300x-amd_06 + - mi300x-amd_07 + - mi300x-amd_08 mi325x: - mi325x-amds_00 - mi325x-amds_01 @@ -288,16 +284,16 @@ labels: - gb300-nv_2 cluster:rtx6000pro-lat: - rtx6000pro-lat_00 - cluster:mi300x-amds: - - mi300x-amds_00 - - mi300x-amds_01 - - mi300x-amds_02 - - mi300x-amds_03 - - mi300x-amds_04 - - mi300x-amds_05 - - mi300x-amds_06 - - mi300x-amds_07 - - mi300x-amds_08 + cluster:mi300x-amd: + - mi300x-amd_00 + - mi300x-amd_01 + - mi300x-amd_02 + - mi300x-amd_03 + - mi300x-amd_04 + - mi300x-amd_05 + - mi300x-amd_06 + - mi300x-amd_07 + - mi300x-amd_08 cluster:mi300x-tw: - mi300x-tw_00 - mi300x-tw_01 @@ -359,8 +355,8 @@ hardware: cluster:rtx6000pro-lat: available-cpu-dram-mib: 1_500_000 gpus-per-node: 8 - cluster:mi300x-amds: - available-cpu-dram-mib: 2_321_924 + cluster:mi300x-amd: + available-cpu-dram-mib: 1_547_820 gpus-per-node: 8 cluster:mi300x-tw: available-cpu-dram-mib: 2_322_328 diff --git a/runners/launch_mi300x-amd.sh b/runners/launch_mi300x-amd.sh new file mode 100755 index 0000000000..d2a656d237 --- /dev/null +++ b/runners/launch_mi300x-amd.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -euo pipefail + +export HF_HUB_CACHE_MOUNT="/raid/inferencex/models/hub" +export AIPERF_MMAP_CACHE_MOUNT="/raid/inferencex/aiperf-mmap-cache" +export AIPERF_DATASET_MMAP_CACHE_DIR="/aiperf_mmap_cache" + +PARTITION="compute" +SQUASH_FILE="/raid/inferencex/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" +LOCK_FILE="${SQUASH_FILE}.lock" + +SPEC_SUFFIX=$([[ "${SPEC_DECODING:-}" == "mtp" ]] && printf '_mtp' || printf '') + +export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}" + +set -x + +JOB_ID=$(set +o pipefail; salloc \ + --partition="$PARTITION" \ + --gres="gpu:$GPU_COUNT" \ + --cpus-per-task=128 \ + --time=180 \ + --no-shell \ + --job-name="$RUNNER_NAME" 2>&1 \ + | tee /dev/stderr \ + | grep -oP 'Granted job allocation \K[0-9]+') + +if [[ -z "$JOB_ID" ]]; then + echo "ERROR: salloc failed to allocate a job" >&2 + exit 1 +fi + +export PORT=$((40000 + (JOB_ID % 10000))) +trap 'scancel "$JOB_ID" 2>/dev/null || true' EXIT + +# Use flock to serialize concurrent imports to the same node-local squash file. +srun --jobid="$JOB_ID" --job-name="$RUNNER_NAME" bash -c " + set -euo pipefail + exec 9>\"$LOCK_FILE\" + flock -w 600 9 || { echo 'Failed to acquire lock for $SQUASH_FILE' >&2; exit 1; } + if unsquashfs -l \"$SQUASH_FILE\" >/dev/null 2>&1; then + echo 'Squash file already exists and is valid, skipping import' + else + rm -f \"$SQUASH_FILE\" + enroot import -o \"$SQUASH_FILE\" docker://$IMAGE + fi +" + +srun --jobid="$JOB_ID" \ + --job-name="$RUNNER_NAME" \ + --container-image="$SQUASH_FILE" \ + --container-mounts="$GITHUB_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE,$AIPERF_MMAP_CACHE_MOUNT:/aiperf_mmap_cache,/dev/kfd:/dev/kfd,/dev/dri:/dev/dri" \ + --container-writable \ + --container-remap-root \ + --container-workdir=/workspace/ \ + --no-container-entrypoint \ + --export=ALL \ + bash "benchmarks/single_node/${SCENARIO_SUBDIR}${EXP_NAME%%_*}_${PRECISION}_mi300x${SPEC_SUFFIX}.sh" + +scancel "$JOB_ID" diff --git a/runners/launch_mi300x-amds.sh b/runners/launch_mi300x-amds.sh deleted file mode 100644 index 96d92e23d7..0000000000 --- a/runners/launch_mi300x-amds.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -export HF_HUB_CACHE_MOUNT="/raid/hf-hub-cache/" - -PARTITION="compute" -SQUASH_DIR="/raid/hf-hub-cache/runtime-cache/enroot" -SQUASH_FILE="$SQUASH_DIR/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" -LOCK_FILE="${SQUASH_FILE}.lock" -COMPUTE_TMPDIR="$SQUASH_DIR/tmp-${UID}" -# Some MI300X compute images do not provision the login user's /run/user/$UID -# directory. Keep Enroot's runtime state node-local instead of inheriting the -# controller's XDG_RUNTIME_DIR, which is not valid on those compute nodes. -export XDG_RUNTIME_DIR="/tmp/enroot-runtime-${UID}" - -# Route spec-decoding=mtp configs to the _mtp benchmark script (parity with -# the h200 launchers, which have carried SPEC_SUFFIX since #392). -SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" ]] && printf '_mtp' || printf '') - -export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}" - -set -x - -# Exclude known-bad nodes; let Slurm pick from anything else: -# chi-mi300x-049: persistent /nvme_home disk-full -# chi-mi300x-121: provisioning incomplete; missing /raid and Enroot storage -JOB_ID=$(set +o pipefail; salloc --partition=$PARTITION --exclude=chi-mi300x-049,chi-mi300x-121 --gres=gpu:$GPU_COUNT --cpus-per-task=256 --time=180 --no-shell --job-name="$RUNNER_NAME" 2>&1 | tee /dev/stderr | grep -oP 'Granted job allocation \K[0-9]+') - -if [ -z "$JOB_ID" ]; then - echo "ERROR: salloc failed to allocate a job" >&2 - exit 1 -fi - -export PORT=$((40000 + (JOB_ID % 10000))) - -# The GitHub runners live on the Slurm controller, whose /home filesystem is -# not mounted on compute nodes. Stage the checked-out source with sbcast, then -# copy only benchmark artifacts back over srun stdout before releasing the job. -LOCAL_WORKSPACE_TAR=$(mktemp "/tmp/inferencex-${JOB_ID}.XXXXXX.tar.gz") -REMOTE_WORKSPACE_TAR="/tmp/inferencex-${JOB_ID}.tar.gz" -REMOTE_WORKSPACE="/tmp/inferencex-${JOB_ID}" - -cleanup() { - local rc=$? - srun --jobid="$JOB_ID" bash -c "rm -rf '$REMOTE_WORKSPACE' '$REMOTE_WORKSPACE_TAR'" >/dev/null 2>&1 || true - scancel "$JOB_ID" 2>/dev/null || true - rm -f "$LOCAL_WORKSPACE_TAR" - exit "$rc" -} -trap cleanup EXIT - -tar -C "$GITHUB_WORKSPACE" \ - --exclude=.git --exclude='*/.git' \ - -czf "$LOCAL_WORKSPACE_TAR" . -sbcast --jobid="$JOB_ID" --force "$LOCAL_WORKSPACE_TAR" "$REMOTE_WORKSPACE_TAR" -srun --jobid="$JOB_ID" --job-name="$RUNNER_NAME" bash -c " - set -euo pipefail - mkdir -p '$XDG_RUNTIME_DIR' - chmod 700 '$XDG_RUNTIME_DIR' - mkdir -p '$COMPUTE_TMPDIR' - chmod 700 '$COMPUTE_TMPDIR' - rm -rf '$REMOTE_WORKSPACE' - mkdir -p '$REMOTE_WORKSPACE' - tar -C '$REMOTE_WORKSPACE' -xzf '$REMOTE_WORKSPACE_TAR' -" - -# Use flock to serialize concurrent imports to the same squash file -srun --jobid="$JOB_ID" --job-name="$RUNNER_NAME" bash -c " - set -euo pipefail - export TMPDIR='$COMPUTE_TMPDIR' - mkdir -p '$SQUASH_DIR' - exec 9>\"$LOCK_FILE\" - flock -w 600 9 || { echo 'Failed to acquire lock for $SQUASH_FILE'; exit 1; } - if unsquashfs -l \"$SQUASH_FILE\" > /dev/null 2>&1; then - echo 'Squash file already exists and is valid, skipping import' - else - rm -f \"$SQUASH_FILE\" - enroot import -o \"$SQUASH_FILE\" docker://$IMAGE - fi -" - -set +e -srun --jobid="$JOB_ID" \ ---container-image=$SQUASH_FILE \ ---container-mounts=$REMOTE_WORKSPACE:/workspace/,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE,/dev/kfd:/dev/kfd,/dev/dri:/dev/dri \ ---container-mount-home \ ---container-writable \ ---container-remap-root \ ---container-workdir=/workspace/ \ ---no-container-entrypoint --export=ALL \ -bash benchmarks/single_node/${SCENARIO_SUBDIR}${EXP_NAME%%_*}_${PRECISION}_mi300x${SPEC_SUFFIX}.sh -BENCHMARK_RC=$? -set -e - -# Stream runtime artifacts back to the controller-side Actions workspace. The -# binary archive is stdout-only; Slurm diagnostics remain on stderr. -srun --jobid="$JOB_ID" bash -c " - set -euo pipefail - cd '$REMOTE_WORKSPACE' - shopt -s nullglob - artifacts=() - for path in results LOGS *.json *.log gpu_metrics.csv profile_*.trace.json.gz eval_results*; do - [[ -e \"\$path\" ]] && artifacts+=(\"\$path\") - done - if (( \${#artifacts[@]} == 0 )); then - tar -czf - --files-from /dev/null - else - tar -czf - \"\${artifacts[@]}\" - fi -" | tar -C "$GITHUB_WORKSPACE" -xzf - - -exit "$BENCHMARK_RC" diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index ddbff934ea..d2e6dc68b0 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -129,7 +129,7 @@ def sample_runner_config(): "cluster:h200-dgxc": {"available-cpu-dram-mib": 1471356, "gpus-per-node": 8}, "cluster:b200-dgxc": {"available-cpu-dram-mib": 3774874, "gpus-per-node": 8}, "cluster:b300-nv": {"available-cpu-dram-mib": 2964436, "gpus-per-node": 8}, - "cluster:mi300x-amds": {"available-cpu-dram-mib": 2321924, "gpus-per-node": 8}, + "cluster:mi300x-amd": {"available-cpu-dram-mib": 1547820, "gpus-per-node": 8}, "cluster:mi355x-amds": {"available-cpu-dram-mib": 3095781, "gpus-per-node": 8}, "cluster:gb200-nv": {"available-cpu-dram-mib": 860160, "gpus-per-node": 4}, }, diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py index ba44514339..7df47fc8be 100644 --- a/utils/matrix_logic/test_validation.py +++ b/utils/matrix_logic/test_validation.py @@ -200,7 +200,7 @@ def valid_runner_config(): "cluster:h100-dgxc": {"available-cpu-dram-mib": 2063837, "gpus-per-node": 8}, "cluster:h200-dgxc": {"available-cpu-dram-mib": 1471356, "gpus-per-node": 8}, "cluster:b200-dgxc": {"available-cpu-dram-mib": 3774874, "gpus-per-node": 8}, - "cluster:mi300x-amds": {"available-cpu-dram-mib": 2321924, "gpus-per-node": 8}, + "cluster:mi300x-amd": {"available-cpu-dram-mib": 1547820, "gpus-per-node": 8}, "cluster:gb200-nv": {"available-cpu-dram-mib": 860160, "gpus-per-node": 4}, }, } diff --git a/utils/test_validate_reusable_sweep_artifacts.py b/utils/test_validate_reusable_sweep_artifacts.py index 69eb633cdc..4eb9923467 100644 --- a/utils/test_validate_reusable_sweep_artifacts.py +++ b/utils/test_validate_reusable_sweep_artifacts.py @@ -402,7 +402,7 @@ def test_eval_validation_uses_logical_runner_from_metadata( tmp_path, 64, logical_runner="mi300x", - physical_runner="mi300x-amds_04", + physical_runner="mi300x-amd_04", ) assert validate_eval_artifacts(tmp_path) == []