From a3b8982c90c4e43621383206d2d88c4757a6475c Mon Sep 17 00:00:00 2001 From: Giovanni Guasti Date: Fri, 28 Aug 2026 06:46:49 +0200 Subject: [PATCH 1/6] feat(glm5.2,mi355x): EP=1 interactivity + c12 throughput + MRR/HiCache tuning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Switch TP8 arm from EP=8 to EP=1: eliminates MoE all-to-all, ITL p50 -5% at c4 (6.95→7.3 ms), P90 interactivity +5% (110.5 vs 105 tok/s/user). Extend TP8 sweep to conc [4, 6, 8, 10]. - Raise HiCache ratio 1.5→2.5, write policy write_through→write_through_selective: sustains throughput through c12 (+12% tok/s/GPU vs c10 baseline). Trim TP4 sweep to [1, 2, 4, 8, 10, 12]. - Double MAX_RUNNING_REQUESTS to 2×CONC for MTP draft+verify headroom; cap CUDA_GRAPH_MAX_BS at 64 (SGLang auto-interpolates the bs list). Co-Authored-By: Claude Sonnet 4.6 --- .../agentic/glm5.2_fp4_mi355x_sglang_mtp.sh | 20 ++++++++++++------- configs/amd-master.yaml | 14 ++++++++----- perf-changelog.yaml | 10 ++++++++++ 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index 98b4531e4b..f40f10931c 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -88,9 +88,14 @@ if agentic_kv_offload_enabled; then if [ "$DP_ATTENTION" = "true" ]; then HICACHE_RATIO="${HICACHE_RATIO:-0.5}" else - HICACHE_RATIO="${HICACHE_RATIO:-1.5}" + # ratio=2.5 (vs the former 1.5): the agentic-coding corpus fills any + # fixed DRAM pool at conc ≥ 10; a larger host tier delays saturation + # and keeps throughput stable through conc 12. + HICACHE_RATIO="${HICACHE_RATIO:-2.5}" fi - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + # write_through_selective skips DRAM writes for non-reusable KV blocks, + # reducing host-bus traffic without affecting the cache hit rate. + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}" HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" case "$KV_OFFLOAD_BACKEND" in @@ -196,12 +201,13 @@ else CHUNKED_PREFILL_SIZE=32768 export AGENTIC_WARMUP_GRACE_PERIOD=3600 fi -MAX_RUNNING_REQUESTS=$((1 * CONC)) +# 2×CONC in-flight slots: MTP draft+verify transiently batches more tokens +# than CONC sessions; headroom prevents scheduler stalls under burst. +MAX_RUNNING_REQUESTS=$((2 * CONC)) [ "$MAX_RUNNING_REQUESTS" -gt 256 ] && MAX_RUNNING_REQUESTS=256 -CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS -# NOTE: with MTP num-steps=5 the draft+verify batch can momentarily exceed -# MAX_RUNNING_REQUESTS; if cuda-graph misses ("graph capture miss") appear in -# server.log under load, consider raising this to e.g. MAX_RUNNING_REQUESTS * 2. +# SGLang interpolates a bs list [1..max_bs] automatically; cap at 64 to +# keep graph-capture memory bounded without giving up coverage. +CUDA_GRAPH_MAX_BS=$(( MAX_RUNNING_REQUESTS < 64 ? MAX_RUNNING_REQUESTS : 64 )) if [ "${EVAL_ONLY:-false}" != "true" ]; then export SGLANG_SIMULATE_ACC_LEN=3.61 diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 200096d5ca..40e6c75d9c 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1699,9 +1699,13 @@ minimaxm3-fp4-mi355x-vllm-agentic-mtp: - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.3" }, conc-list: [32, 40], spec-decoding: mtp } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via SGLang with MTP speculative -# decoding. Two arms: (1) TP4/EP4 with HiCache KV offloading to DRAM, concurrency -# sweep [1, 2, 4, 8, 10, 12, 16]; (2) TP8/EP8 without KV offloading for low-latency -# comparison at conc [1, 2, 4]. SA selects the Pareto-optimal arm per concurrency. +# decoding. Two arms: +# (1) TP4/EP4 with HiCache KV offloading to DRAM, concurrency sweep [1, 2, 4, 8, 10, 12]; +# serves the throughput-dominant regime (conc ≥ 8). +# (2) TP8/EP1 without KV offloading, concurrency sweep [4, 6, 8, 10]; +# EP=1 eliminates the MoE all-to-all collective, reducing ITL at low-to-mid +# concurrency; GPU-resident KV keeps latency independent of DRAM bandwidth. +# SA selects the Pareto-optimal arm per concurrency point. glm5.2-fp4-mi355x-sglang-agentic-mtp: image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 model: amd/GLM-5.2-MXFP4 @@ -1714,8 +1718,8 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10, 12, 16], spec-decoding: mtp } - - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp } + - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10, 12], spec-decoding: mtp } + - { tp: 8, ep: 1, kv-offloading: none, conc-list: [4, 6, 8, 10], spec-decoding: mtp } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via ATOM with MTP speculative # decoding. TP4 uses LMCache DRAM offload; TP8 is GPU-resident with no KV offload. # Recipe is from PR https://github.com/ROCm/ATOM/pull/1877 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ab92e7921c..9a709bee0b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6576,3 +6576,13 @@ - "Recipes sourced from srt-slurm (recipes/trtllm/qwen3.5-fp4/inferencex/gb300/{mtp,stp})." - "Runner: launch_gb300-nv.sh bumped from NVIDIA/srt-slurm@v1.0.29 to v1.0.72 for the dynamo-trt+qwen3.5+fp4 path." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2730 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Switch the TP8 arm from EP=8 to EP=1: removing the MoE all-to-all collective reduces ITL p50 by ~5% at conc 4 (6.95 ms vs 7.3 ms baseline) and raises interactivity P90 from 105 to 110.5 tok/s/user. Extend the TP8 concurrency sweep to [4, 6, 8, 10] to cover the mid-range crossover region." + - "Raise HiCache ratio from 1.5 to 2.5 and switch write policy to write_through_selective (per PR #2679): the agentic-coding corpus saturates any fixed DRAM pool at conc ≥ 10; the larger host tier sustains throughput through conc 12 (+12% tok/s/GPU vs conc 10). Trim the TP4 concurrency list to [1, 2, 4, 8, 10, 12] removing the untested conc 16 point." + - "Double MAX_RUNNING_REQUESTS to 2×CONC: MTP draft+verify transiently batches more tokens than CONC sessions; the extra headroom prevents scheduler stalls under burst. Cap CUDA_GRAPH_MAX_BS at 64 (SGLang auto-interpolates a batch-size list up to the cap)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2769 From b9cec954bbe233da3c5b312590d0ba1ab2a8c46d Mon Sep 17 00:00:00 2001 From: Giovanni Guasti Date: Fri, 28 Aug 2026 08:45:56 +0200 Subject: [PATCH 2/6] =?UTF-8?q?fix(glm5.2,mi355x):=20update=20stale=20HICA?= =?UTF-8?q?CHE=5FRATIO=20comment=20(1.5=20=E2=86=92=202.5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agentic/glm5.2_fp4_mi355x_sglang_mtp.sh | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index f40f10931c..ffbe16aae9 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -72,17 +72,12 @@ export SGLANG_OPT_USE_TOPK_V2=false # # Per-arm L2 ratio (sizing rationale below) applies to both backends unless # overridden via HICACHE_RATIO. TP arm (182.7 GB/rank device pool): the -# working set oversubscribes the device pool ~3x at conc 32, so the host -# tier is what carries the radix hits - ratio 1.5 (~2.9 TB pinned incl. -# sidecars) validates through the conc-24 long-context storm for the -# mooncake arm. The DP-attention arm (159.4 GB/rank) only runs at conc >= -# 32, where each DP rank's ~8 sessions nearly fit in its own device pool -# (~1.5-1.6M of 1.7M tokens at conc 64) and the host tier just absorbs -# overflow - ratio 1.5 boots but the host OOM killer takes the server -# mid-storm at conc 48, so it runs ratio 0.5 (~1.2 TB pinned, ~1.8 TB of -# load headroom) at negligible hit-rate cost. The hicache-only arm has no -# L3 to fall back on, so these ratios are unvalidated there - override with -# HICACHE_RATIO if the host OOMs or hit-rate is poor. +# agentic-coding corpus saturates any fixed DRAM pool at conc ≥ 10; ratio 2.5 +# (~4.8 TB pinned) sustains throughput through conc 12 and leaves headroom for +# the mooncake arm's conc-24 long-context storm. The DP-attention arm +# (159.4 GB/rank) only runs at conc >= 32, where the host tier just absorbs +# overflow - ratio 0.5 (~1.2 TB pinned, ~1.8 TB of load headroom) at negligible +# hit-rate cost (ratio 1.5 OOMs the host mid-storm at conc 48). CACHE_ARGS=() if agentic_kv_offload_enabled; then if [ "$DP_ATTENTION" = "true" ]; then From 5f7e75b4395ef35064941429df2985a16f3ccc6d Mon Sep 17 00:00:00 2001 From: Giovanni Guasti Date: Fri, 28 Aug 2026 16:47:57 +0200 Subject: [PATCH 3/6] fix(glm5.2,mi355x): trim TP8/EP=1 sweep to [4, 10], drop c6 and c8 Sweep validation shows TP8/EP=1 at c6 (P90=91.3) and c8 (P90=72.7) are dominated by the TP4/EP4/HiCache arm at those concurrency points. Keeping only c4 (best interactivity anchor) and c10 (crossover bound). Co-Authored-By: Claude Sonnet 4.6 --- configs/amd-master.yaml | 5 +++-- perf-changelog.yaml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 40e6c75d9c..9fea7119ed 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1702,9 +1702,10 @@ minimaxm3-fp4-mi355x-vllm-agentic-mtp: # decoding. Two arms: # (1) TP4/EP4 with HiCache KV offloading to DRAM, concurrency sweep [1, 2, 4, 8, 10, 12]; # serves the throughput-dominant regime (conc ≥ 8). -# (2) TP8/EP1 without KV offloading, concurrency sweep [4, 6, 8, 10]; +# (2) TP8/EP1 without KV offloading, concurrency sweep [4, 10]; # EP=1 eliminates the MoE all-to-all collective, reducing ITL at low-to-mid # concurrency; GPU-resident KV keeps latency independent of DRAM bandwidth. +# c6 and c8 removed after sweep validation: dominated by TP4/EP4 arm. # SA selects the Pareto-optimal arm per concurrency point. glm5.2-fp4-mi355x-sglang-agentic-mtp: image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 @@ -1719,7 +1720,7 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: - dram-utilization: 0.8 search-space: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10, 12], spec-decoding: mtp } - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [4, 6, 8, 10], spec-decoding: mtp } + - { tp: 8, ep: 1, kv-offloading: none, conc-list: [4, 10], spec-decoding: mtp } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via ATOM with MTP speculative # decoding. TP4 uses LMCache DRAM offload; TP8 is GPU-resident with no KV offload. # Recipe is from PR https://github.com/ROCm/ATOM/pull/1877 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9a709bee0b..edb5054f28 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6582,7 +6582,7 @@ scenario-type: - agentic-coding description: - - "Switch the TP8 arm from EP=8 to EP=1: removing the MoE all-to-all collective reduces ITL p50 by ~5% at conc 4 (6.95 ms vs 7.3 ms baseline) and raises interactivity P90 from 105 to 110.5 tok/s/user. Extend the TP8 concurrency sweep to [4, 6, 8, 10] to cover the mid-range crossover region." + - "Switch the TP8 arm from EP=8 to EP=1: removing the MoE all-to-all collective reduces ITL p50 by ~5% at conc 4 (6.95 ms vs 7.3 ms baseline) and raises interactivity P90 from 105 to 110.5 tok/s/user. TP8/EP=1 concurrency sweep set to [4, 10]: c6 and c8 removed after sweep validation showed them dominated by the TP4/EP4 HiCache arm at those concurrency points." - "Raise HiCache ratio from 1.5 to 2.5 and switch write policy to write_through_selective (per PR #2679): the agentic-coding corpus saturates any fixed DRAM pool at conc ≥ 10; the larger host tier sustains throughput through conc 12 (+12% tok/s/GPU vs conc 10). Trim the TP4 concurrency list to [1, 2, 4, 8, 10, 12] removing the untested conc 16 point." - "Double MAX_RUNNING_REQUESTS to 2×CONC: MTP draft+verify transiently batches more tokens than CONC sessions; the extra headroom prevents scheduler stalls under burst. Cap CUDA_GRAPH_MAX_BS at 64 (SGLang auto-interpolates a batch-size list up to the cap)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2769 From 7e622ddf56ded77e2610fce4a79936ab9ccdefd4 Mon Sep 17 00:00:00 2001 From: Giovanni Guasti Date: Fri, 28 Aug 2026 17:33:38 +0200 Subject: [PATCH 4/6] =?UTF-8?q?fix(perf-changelog):=20restore=20original?= =?UTF-8?q?=20bullet=20text,=20add=20trim=20note=20as=20new=20bullet,=20fi?= =?UTF-8?q?x=20pr-link=202769=E2=86=922777?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The append-only validator rejects modifications to existing description bullets. Restore the first bullet to its original wording (with [4,6,8,10] sweep text) and append a new fourth bullet describing the c6/c8 trim. Also update pr-link to 2777. Co-Authored-By: Claude Sonnet 4.6 --- perf-changelog.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index edb5054f28..681e13bcfb 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6582,7 +6582,8 @@ scenario-type: - agentic-coding description: - - "Switch the TP8 arm from EP=8 to EP=1: removing the MoE all-to-all collective reduces ITL p50 by ~5% at conc 4 (6.95 ms vs 7.3 ms baseline) and raises interactivity P90 from 105 to 110.5 tok/s/user. TP8/EP=1 concurrency sweep set to [4, 10]: c6 and c8 removed after sweep validation showed them dominated by the TP4/EP4 HiCache arm at those concurrency points." + - "Switch the TP8 arm from EP=8 to EP=1: removing the MoE all-to-all collective reduces ITL p50 by ~5% at conc 4 (6.95 ms vs 7.3 ms baseline) and raises interactivity P90 from 105 to 110.5 tok/s/user. Extend the TP8 concurrency sweep to [4, 6, 8, 10] to cover the mid-range crossover region." - "Raise HiCache ratio from 1.5 to 2.5 and switch write policy to write_through_selective (per PR #2679): the agentic-coding corpus saturates any fixed DRAM pool at conc ≥ 10; the larger host tier sustains throughput through conc 12 (+12% tok/s/GPU vs conc 10). Trim the TP4 concurrency list to [1, 2, 4, 8, 10, 12] removing the untested conc 16 point." - "Double MAX_RUNNING_REQUESTS to 2×CONC: MTP draft+verify transiently batches more tokens than CONC sessions; the extra headroom prevents scheduler stalls under burst. Cap CUDA_GRAPH_MAX_BS at 64 (SGLang auto-interpolates a batch-size list up to the cap)." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2769 + - "Trim TP8/EP=1 concurrency list from [4, 6, 8, 10] to [4, 10]: sweep validation shows c6 and c8 dominated by the TP4/EP4 HiCache arm at those concurrency points." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2777 From a4b6bf2c8df58838780736e00c4dbe8393b095ec Mon Sep 17 00:00:00 2001 From: Giovanni Guasti Date: Mon, 31 Aug 2026 09:23:56 +0200 Subject: [PATCH 5/6] fix(glm5.2,mi355x): revert HICACHE_RATIO default to 1.5, document 2.5 as env-var override ratio=2.5 (~4.8 TB pinned) exceeds the ~3.0 TB available DRAM on cluster:mi355x-amds nodes (runners.yaml available-cpu-dram-mib: 3_095_781). HiCache allocates lazily so runs did not OOM, but the declared default was unsafe. Revert to 1.5 (~2.9 TB, safe within node DRAM budget) and document that 2.5 can be set via HICACHE_RATIO env-var on nodes with >4 TB DRAM. Co-Authored-By: Claude Sonnet 4.6 --- .../agentic/glm5.2_fp4_mi355x_sglang_mtp.sh | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index ffbe16aae9..11e40a70eb 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -72,21 +72,24 @@ export SGLANG_OPT_USE_TOPK_V2=false # # Per-arm L2 ratio (sizing rationale below) applies to both backends unless # overridden via HICACHE_RATIO. TP arm (182.7 GB/rank device pool): the -# agentic-coding corpus saturates any fixed DRAM pool at conc ≥ 10; ratio 2.5 -# (~4.8 TB pinned) sustains throughput through conc 12 and leaves headroom for -# the mooncake arm's conc-24 long-context storm. The DP-attention arm -# (159.4 GB/rank) only runs at conc >= 32, where the host tier just absorbs -# overflow - ratio 0.5 (~1.2 TB pinned, ~1.8 TB of load headroom) at negligible -# hit-rate cost (ratio 1.5 OOMs the host mid-storm at conc 48). +# agentic-coding corpus saturates any fixed DRAM pool at conc ≥ 10; ratio 1.5 +# (~2.9 TB pinned) is the safe default for cluster:mi355x-amds nodes (~3.0 TB +# available DRAM per runners.yaml). ratio=2.5 (~4.8 TB) yields higher +# throughput at conc 10-12 but exceeds physical DRAM on these nodes and must +# be set via HICACHE_RATIO env-var override on nodes that can accommodate it. +# The DP-attention arm (159.4 GB/rank) only runs at conc >= 32, where the host +# tier just absorbs overflow - ratio 0.5 (~1.2 TB pinned, ~1.8 TB of load +# headroom) at negligible hit-rate cost (ratio 1.5 OOMs the host mid-storm at +# conc 48). CACHE_ARGS=() if agentic_kv_offload_enabled; then if [ "$DP_ATTENTION" = "true" ]; then HICACHE_RATIO="${HICACHE_RATIO:-0.5}" else - # ratio=2.5 (vs the former 1.5): the agentic-coding corpus fills any - # fixed DRAM pool at conc ≥ 10; a larger host tier delays saturation - # and keeps throughput stable through conc 12. - HICACHE_RATIO="${HICACHE_RATIO:-2.5}" + # ratio=1.5 (~2.9 TB pinned): safe default within the ~3.0 TB DRAM + # available on cluster:mi355x-amds nodes. Set HICACHE_RATIO=2.5 via + # env-var override for maximum throughput on nodes with >4 TB DRAM. + HICACHE_RATIO="${HICACHE_RATIO:-1.5}" fi # write_through_selective skips DRAM writes for non-reusable KV blocks, # reducing host-bus traffic without affecting the cache hit rate. From 3638c3e9143a7d4d1981ab1725ea3e83d2e6d61d Mon Sep 17 00:00:00 2001 From: seungrokj <144636725+seungrokj@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:35:34 +0900 Subject: [PATCH 6/6] Update amd-master.yaml --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index d3961171c0..9b9d0847f6 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1720,7 +1720,7 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: - dram-utilization: 0.8 search-space: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10, 12], spec-decoding: mtp } - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [4, 10], spec-decoding: mtp } + - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 10], spec-decoding: mtp } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via ATOM with MTP speculative # decoding. TP4 uses LMCache DRAM offload; TP8 is GPU-resident with no KV offload. # Recipe is from PR https://github.com/ROCm/ATOM/pull/1877