diff --git a/.settings.yaml b/.settings.yaml index 49161095c..86790489f 100644 --- a/.settings.yaml +++ b/.settings.yaml @@ -204,7 +204,7 @@ testing: # GPU CI runtime pins consumed through .github/actions/load-versions. # renovate: datasource=helm depName=gpu-operator depType=testing - gpu_operator_chart_version: 'v26.3.3' + gpu_operator_chart_version: 'v26.7.0' # Base image for the CI smoke-test snapshot-agent image. The agent binary is # static Go and detects GPUs driver-free via NFD/PCI, so it no longer needs a # CUDA base / nvidia-smi. Matches the NVIDIA static distroless base used for diff --git a/demos/query.md b/demos/query.md index f98c23bcd..32c6f7ede 100644 --- a/demos/query.md +++ b/demos/query.md @@ -50,10 +50,10 @@ aicr query \ ```yaml enabled: true +kernelModuleType: auto maxParallelUpgrades: 5 rdma: enabled: false -useOpenKernelModules: true version: 580.173.02 ``` diff --git a/docs/integrator/aks-gpu-setup.md b/docs/integrator/aks-gpu-setup.md index 9f57fc9e1..83bf850b5 100644 --- a/docs/integrator/aks-gpu-setup.md +++ b/docs/integrator/aks-gpu-setup.md @@ -399,7 +399,7 @@ required so the device plugin's volume-mounts allocation strategy still works for legitimately allocated pods, but it leaves the **volume-mounts device-request path open** (a pod that declares a `/dev/null`-backed mount whose destination is under `/var/run/nvidia-container-devices` can still select -devices — the pinned toolkit v1.19.1 accepts the volume-mount device request +devices — the pinned toolkit v1.20.0 accepts the volume-mount device request only when the mount *source* is `/dev/null`). This is the same posture as GPU-Operator-managed mode — not a regression — but it means full multi-tenant isolation additionally requires an admission policy restricting a diff --git a/docs/integrator/automation.md b/docs/integrator/automation.md index a6d5e6585..1fd53d47f 100644 --- a/docs/integrator/automation.md +++ b/docs/integrator/automation.md @@ -231,7 +231,7 @@ spec: # Helm chart from upstream - repoURL: https://helm.ngc.nvidia.com/nvidia chart: gpu-operator - targetRevision: v26.3.3 + targetRevision: v26.7.0 helm: valueFiles: # Values live under the numbered bundle dir (NNN-/) diff --git a/docs/integrator/data-flow.md b/docs/integrator/data-flow.md index 833eb0453..b0e7007fc 100644 --- a/docs/integrator/data-flow.md +++ b/docs/integrator/data-flow.md @@ -634,7 +634,7 @@ spec: sources: # Helm chart from upstream - repoURL: https://helm.ngc.nvidia.com/nvidia - targetRevision: v26.3.3 + targetRevision: v26.7.0 chart: gpu-operator helm: valueFiles: diff --git a/docs/integrator/recipe-development.md b/docs/integrator/recipe-development.md index 4985364b6..a469decc3 100644 --- a/docs/integrator/recipe-development.md +++ b/docs/integrator/recipe-development.md @@ -888,7 +888,7 @@ spec: # recipes/registry.yaml - name: gpu-operator helm: - defaultVersion: v26.3.3 # Changed from v26.3.2 + defaultVersion: v26.7.0 # Changed from v26.3.3 ``` **Adding components:** diff --git a/docs/user/air-gap-mirror.md b/docs/user/air-gap-mirror.md index e495b804d..43f2544d3 100644 --- a/docs/user/air-gap-mirror.md +++ b/docs/user/air-gap-mirror.md @@ -112,7 +112,7 @@ metadata: name: aicr-images spec: images: - - name: nvcr.io/nvidia/gpu-operator:v26.3.3 + - name: nvcr.io/nvidia/gpu-operator:v26.7.0 - name: registry.k8s.io/nfd/node-feature-discovery:v0.19.0 # ... --- @@ -124,7 +124,7 @@ spec: charts: - name: gpu-operator repoURL: oci://ghcr.io/nvidia - version: v26.3.3 + version: v26.7.0 # ... ``` @@ -187,13 +187,13 @@ components: - name: aicr-images required: true images: - - nvcr.io/nvidia/gpu-operator:v26.3.3 + - nvcr.io/nvidia/gpu-operator:v26.7.0 - registry.k8s.io/nfd/node-feature-discovery:v0.19.0 # ... charts: - name: gpu-operator url: oci://ghcr.io/nvidia/gpu-operator - version: v26.3.3 + version: v26.7.0 namespace: gpu-operator # ... ``` diff --git a/docs/user/api-reference.md b/docs/user/api-reference.md index 2b875303a..c6d75c840 100644 --- a/docs/user/api-reference.md +++ b/docs/user/api-reference.md @@ -714,7 +714,7 @@ curl -X POST "http://localhost:8080/v1/bundle" \ "apiVersion": "aicr.run/v1alpha2", "kind": "RecipeResult", "componentRefs": [ - {"name": "gpu-operator", "type": "Helm", "chart": "gpu-operator", "source": "https://helm.ngc.nvidia.com/nvidia", "version": "v26.3.3", "namespace": "gpu-operator", "valuesFile": "components/gpu-operator/values.yaml"}, + {"name": "gpu-operator", "type": "Helm", "chart": "gpu-operator", "source": "https://helm.ngc.nvidia.com/nvidia", "version": "v26.7.0", "namespace": "gpu-operator", "valuesFile": "components/gpu-operator/values.yaml"}, {"name": "network-operator", "type": "Helm", "chart": "network-operator", "source": "https://helm.ngc.nvidia.com/nvidia", "version": "26.1.1", "namespace": "nvidia-network-operator", "valuesFile": "components/network-operator/values.yaml"} ], "deploymentOrder": ["gpu-operator", "network-operator"] diff --git a/docs/user/cli-reference.md b/docs/user/cli-reference.md index f957a7c04..a09316b46 100644 --- a/docs/user/cli-reference.md +++ b/docs/user/cli-reference.md @@ -2115,7 +2115,7 @@ The `--vendor-charts` flag pulls upstream Helm chart bytes into the bundle at bu my-bundle/ 001-gpu-operator/ Chart.yaml # wrapper, declares the vendored subchart - charts/gpu-operator-v26.3.3.tgz # vendored upstream tarball + charts/gpu-operator-v26.7.0.tgz # vendored upstream tarball values.yaml # values nested under the subchart name cluster-values.yaml # dynamic values, also nested install.sh # helm upgrade --install ./ ... @@ -2144,10 +2144,10 @@ kind: BundleProvenance vendoredCharts: - name: gpu-operator chart: gpu-operator - version: v26.3.3 + version: v26.7.0 repository: https://helm.ngc.nvidia.com/nvidia sha256: abc123... - tarballName: gpu-operator-v26.3.3.tgz + tarballName: gpu-operator-v26.7.0.tgz pullerVersion: helm-cli v3.20.2 ``` diff --git a/docs/user/container-images.md b/docs/user/container-images.md index a617f14b9..1c476cadb 100644 --- a/docs/user/container-images.md +++ b/docs/user/container-images.md @@ -42,7 +42,7 @@ _Rendering fidelity:_ `catalog-parity: charts are rendered with the shared recip | gatekeeper | helm | gatekeeper/gatekeeper | 3.22.2 | 3 | | gcp-driver-installer | manifest | — | — | 3 | | gke-nccl-tcpxo | manifest | — | — | 4 | -| gpu-operator | helm | nvidia/gpu-operator | v26.3.3 | 15 | +| gpu-operator | helm | nvidia/gpu-operator | v26.7.0 | 15 | | gpu-operator-ocp | manifest | — | — | 0 | | gpu-operator-ocp-olm | manifest | — | — | 0 | | grove | helm | grove-charts | v0.1.0-alpha.12 | 1 | @@ -64,8 +64,8 @@ _Rendering fidelity:_ `catalog-parity: charts are rendered with the shared recip | nfd-ocp-olm | manifest | — | — | 0 | | nodewright-customizations | manifest | — | — | 5 | | nodewright-operator | helm | nodewright | v0.17.1 | 3 | -| nvidia-dra-driver-gpu | helm | dra-driver-nvidia-gpu | 0.4.1 | 1 | -| nvidia-dra-driver-gpu-ocp | helm | dra-driver-nvidia-gpu | 0.4.1 | 1 | +| nvidia-dra-driver-gpu | helm | dra-driver-nvidia-gpu | 0.5.0 | 1 | +| nvidia-dra-driver-gpu-ocp | helm | dra-driver-nvidia-gpu | 0.5.0 | 1 | | nvsentinel | helm | nvsentinel | v1.20.0 | 6 | | prometheus-adapter | helm | prometheus-community/prometheus-adapter | 5.3.0 | 1 | | prometheus-adapter-ocp | helm | prometheus-community/prometheus-adapter | 5.3.0 | 1 | @@ -149,20 +149,20 @@ _No images extracted._ ### gpu-operator - `docker.io/library/busybox:1.38.0@sha256:dc2d74b28e4cf8984fa52af1f39bc7c3d9c73760b41a74d629f5d11b1ab28616` -- `nvcr.io/nvidia/cloud-native/dcgm:4.5.2-1-ubuntu22.04` -- `nvcr.io/nvidia/cloud-native/gdrdrv:v2.5.2` -- `nvcr.io/nvidia/cloud-native/k8s-cc-manager:v0.4.0` -- `nvcr.io/nvidia/cloud-native/k8s-driver-manager:v0.11.0` -- `nvcr.io/nvidia/cloud-native/k8s-mig-manager:v0.14.2` -- `nvcr.io/nvidia/cloud-native/nvidia-fs:2.27.3` -- `nvcr.io/nvidia/cloud-native/nvidia-sandbox-device-plugin:v0.0.3` -- `nvcr.io/nvidia/cloud-native/vgpu-device-manager:v0.4.2` +- `nvcr.io/nvidia/cloud-native/dcgm:4.6.0-1-ubuntu24.04` +- `nvcr.io/nvidia/cloud-native/gdrdrv:v2.6` +- `nvcr.io/nvidia/cloud-native/k8s-cc-manager:v0.4.3` +- `nvcr.io/nvidia/cloud-native/k8s-driver-manager:v0.12.0` +- `nvcr.io/nvidia/cloud-native/k8s-mig-manager:v0.15.0` +- `nvcr.io/nvidia/cloud-native/nvidia-fs:2.29.4` +- `nvcr.io/nvidia/cloud-native/nvidia-sandbox-device-plugin:v0.0.5` +- `nvcr.io/nvidia/cloud-native/vgpu-device-manager:v0.5.0` - `nvcr.io/nvidia/driver:580.173.02` -- `nvcr.io/nvidia/gpu-operator:v26.3.3` -- `nvcr.io/nvidia/k8s-device-plugin:v0.19.3` -- `nvcr.io/nvidia/k8s/container-toolkit:v1.19.1` -- `nvcr.io/nvidia/k8s/dcgm-exporter:4.5.3-4.8.2-distroless` -- `nvcr.io/nvidia/kubevirt-gpu-device-plugin:v1.5.0` +- `nvcr.io/nvidia/gpu-operator:v26.7.0` +- `nvcr.io/nvidia/k8s-device-plugin:v0.20.0` +- `nvcr.io/nvidia/k8s/container-toolkit:v1.20.0` +- `nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless` +- `nvcr.io/nvidia/kubevirt-gpu-device-plugin:v1.6.0` ### gpu-operator-ocp @@ -279,11 +279,11 @@ _No images extracted._ ### nvidia-dra-driver-gpu -- `registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.4.1` +- `registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.5.0` ### nvidia-dra-driver-gpu-ocp -- `registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.4.1` +- `registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.5.0` ### nvsentinel @@ -448,7 +448,7 @@ Y = artifact attached, - = artifact absent, ? = could not probe. Image Sig SBOM Prov Notes -------------------------------------------------------------- --- ---- ---- ----- - registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.4.1 Y - - + registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.5.0 Y - - Summary: 1/1 signed · 0/1 SBOM · 0/1 provenance ``` diff --git a/examples/recipes/aks-training.yaml b/examples/recipes/aks-training.yaml index 418788d30..cdd8d4dcc 100644 --- a/examples/recipes/aks-training.yaml +++ b/examples/recipes/aks-training.yaml @@ -52,7 +52,7 @@ componentRefs: chart: gpu-operator type: Helm source: https://helm.ngc.nvidia.com/nvidia - version: v26.3.3 + version: v26.7.0 valuesFile: components/gpu-operator/values-aks-training.yaml dependencyRefs: - nfd @@ -151,7 +151,7 @@ componentRefs: chart: dra-driver-nvidia-gpu type: Helm source: oci://registry.k8s.io/dra-driver-nvidia/charts - version: 0.4.1 + version: 0.5.0 valuesFile: components/nvidia-dra-driver-gpu/values.yaml overrides: controller: diff --git a/examples/recipes/eks-gb200-ubuntu-training-with-validation.yaml b/examples/recipes/eks-gb200-ubuntu-training-with-validation.yaml index c9b91a3f8..49bbe8429 100644 --- a/examples/recipes/eks-gb200-ubuntu-training-with-validation.yaml +++ b/examples/recipes/eks-gb200-ubuntu-training-with-validation.yaml @@ -61,9 +61,9 @@ validation: deployment: constraints: - name: gpu-operator.version - value: "== v26.3.3" + value: "== v26.7.0" severity: warning - remediation: "Update GPU Operator to v26.3.3" + remediation: "Update GPU Operator to v26.7.0" checks: - expected-resources @@ -100,7 +100,7 @@ componentRefs: chart: gpu-operator type: Helm source: https://helm.ngc.nvidia.com/nvidia - version: v26.3.3 + version: v26.7.0 valuesFile: components/gpu-operator/values-eks-training.yaml expectedResources: - kind: Deployment @@ -129,7 +129,7 @@ componentRefs: chart: dra-driver-nvidia-gpu type: Helm source: oci://registry.k8s.io/dra-driver-nvidia/charts - version: 0.4.1 + version: 0.5.0 valuesFile: components/nvidia-dra-driver-gpu/values.yaml dependencyRefs: - gpu-operator diff --git a/examples/recipes/eks-training.yaml b/examples/recipes/eks-training.yaml index e4dc850a9..8de030f01 100644 --- a/examples/recipes/eks-training.yaml +++ b/examples/recipes/eks-training.yaml @@ -41,7 +41,7 @@ componentRefs: chart: gpu-operator type: Helm source: https://helm.ngc.nvidia.com/nvidia - version: v26.3.3 + version: v26.7.0 valuesFile: components/gpu-operator/values-eks-training.yaml dependencyRefs: - cert-manager diff --git a/pkg/bundler/testdata/stock_render_golden.yaml b/pkg/bundler/testdata/stock_render_golden.yaml index 61a272c15..71757c70d 100644 --- a/pkg/bundler/testdata/stock_render_golden.yaml +++ b/pkg/bundler/testdata/stock_render_golden.yaml @@ -3,52 +3,52 @@ # # One entry per leaf overlay: a digest over its fully rendered helm-deployer # bundle tree (sorted relative paths paired with per-file content hashes). -a100-aks-ubuntu-training-kubeflow: b8161ed651945695164e296dbda3465d288ea19dcdf8d0ef6e84a1cc92af5cdd -a100-any: 8f6e700544dfd93a8be31a148badb32829924807aa1d13c2db2b80527e94c365 -a100-eks-ubuntu-training-kubeflow: 9d87274b22b1e3a3c35cee9f884ee00aa8bf2d98c75c7961be6d055ca8b2b4c4 -a100-gke-cos-training-kubeflow: 5bfae64e203fdc096e7e5df3d912d45fbf98e79543248c380c73e59562e059f6 -a100-oke-ubuntu-training-kubeflow: 5c46cfdaf69b6f38d2b3b377fc5641b5aa6c84b1030fe97426dc4874ed0a5819 -b200-any: b0a2e105824bd5042396bbdc91529a5986a62d7e0e2be5fd41a4ee31bc3ab24d -b200-gke-cos-inference-dynamo: 788a4749c063fab956cb133271aaec135148171fc761a34faf22e207d0356924 -b200-gke-cos-training-kubeflow: 0d72edc6fe724e87aaa278a60e31fd852b995b978ed7c7768b1ea1ed8cb71864 -bcm-inference: 0de8f37c9026519148af9d1f915e0cb35cef0799d39eb19a9f10736f54170742 -gb200-any: 5523fb162d04b85010ea14adcacde848823051530ecb2e0784d27bc5c0e10e1a -gb200-eks-ubuntu-inference-dynamo: f306a1fdd7d5fa81cdac39486928c0ebebf46690318d3279bd8a1258f4132698 -gb200-eks-ubuntu-training-kubeflow: f9de625240b1836d2d940cef2da8b989bd42d78e73273230ea7bd1b693345f9f -gb200-eks-ubuntu-training-slurm: 52c6c3858c36742d53f23ee36c5cb47859ef5b61ff2c9bbc49e932570b617f62 -gb200-oke-ubuntu-inference-dynamo: 4a8fa3f14fd6f8023311baccc03d12405d75d52b96159959a771ebca699e2cf6 -gb200-oke-ubuntu-training-kubeflow: 090e678f7ff902fdce9a06afcb5b30aea11a4b90c86979a6ca9059fbd9c14605 -gb300-any: 357198d4ae6a0e73528cf0ceda23258213e7eddba5f83aa25f133562bdaf65b0 -gb300-eks-ubuntu-inference-dynamo: f20aca19f018e8bbb9970276a6413c6cc898881ceb647c33c0ab7b8a9b078f3e -gb300-eks-ubuntu-training-kubeflow: 0d6b69f81dd4919d02ebad13324b5a1efe3b3efdb783bf91f5454eca194c57fc -h100-aks-ubuntu-inference-dynamo: 5b53e936929ee7ed8f07256472a2ec6885b7f0c752d003ce22a98b258fded50b -h100-aks-ubuntu-training-kubeflow: 5f5d9d468a54571bded9030a968a684a43779161dd0a487521283355bd19560d -h100-aks-ubuntu-training-slurm: 8544a761f71c8577952126a19c9a973b0890826d1680c89ec2ff63540d1b1ef6 -h100-any: 9a1e512c6d7fec5f9920ffc2d4b1f036fe59ba80322d4de286d931685bac70d5 -h100-bcm-ubuntu-training: cba1fe3491fe878683a4a99fe0af5bf08673b3d4ad21669351d6ce719b313ca3 -h100-eks-ubuntu-inference-dynamo: c42f8d3e2bad0aecaa952cf84a41e3603148c2ed2f7163c79c67397fca1ed187 -h100-eks-ubuntu-inference-nim: 6b6446a298885359b76f697970c33b80ace95de9cc0a4ab549fe58b8e579ec80 -h100-eks-ubuntu-training-kubeflow: e78076ebfd48bec60d410a3f0aa3da2d1cdff0ebec3f506b8b87e07110fd92f4 -h100-eks-ubuntu-training-slurm: f42961b910bec0b36b2b6ffcd5e2df6ec54f11755b94190b901884dc7cbff996 -h100-gke-cos-inference-dynamo: 1cd1373905cde68c6781925e354459c66e4094c8e6ede458bb75a377628fb5cb -h100-gke-cos-training-kubeflow: 6f0844b970b3706e3b5eb70dc3ae79c17119cf66c3127a7368813d345467bb83 -h100-gke-cos-training-slurm: edbbfa71d6dd096bf32d73412929f5779eafd5e172c990839098e02663a7b88d -h100-kind-inference-dynamo: b1734f6c44bff5e21da1d67b9f3a1b3d2de4fd6b48db411a19208c8bf7fda62d -h100-kind-training-kubeflow: 1c589e48629c86e5c8b7db3de708fb502bed7a4e48468a635e5da92cb83c3c1d -h100-kind-training-slurm: c8f780ca26d66c5191ba97409fdf9f49c5226eb78fee6ed54e06d05490b2215a -h200-any: 8f30849eac9a8489ce02ea195d538f6f76ce8afb0bcf1c8515843869bd304024 -h200-eks-inference: dbbeb66fc64382271916197fb24d951d10a5d4972c59aa7931438e5e3ec151f0 -h200-eks-training: 14db15267d453edbe3e4121fd1933cae633f2377c3aa6c198e97dc076f8cd464 -l40-any: da7bb3a2831930ddb224927d3ebfb5a5dd65d3c9a128c632aad413fb29c0e28b -l40s-any: 8c2f0d3ff1e2f0b5df3869650d3e74ec3808484f52949726bfd5db9c409c8199 -l40s-oke-inference: 69696a6ec37fc930d5fe728d28f979badb9ef9f6a4fa45a39a58b99c981c93ec -l40s-oke-training: 4afd8779e7c706530cd2936fdf0ce7537f86c9cbc019e02c4e05a6787cb31310 -monitoring-hpa: 1e4f15813317d9b472f66cedd8a280a1555a0a7e3142ba9c3dbfb014e578fbcd -ocp-inference-nim: 67cb04b6ad946fe56ce423972ec0692ebe8ad66f571f29e94536ae0124eee91c -ocp-training: 1082d9d2f5b8dd302d498ab9cb6c936dd6f95996f65e5ce41746ff84aefed04e -rtx-pro-6000-any: ca387e660b770165b828770102eaa0396b8a51f7157d78a2c714c631d5ecf70d -rtx-pro-6000-eks-ubuntu-inference-dynamo: 285ad9f64afc85fb0cf58269e18531a2457299453e35f1c0d9608f3a4f08288d -rtx-pro-6000-eks-ubuntu-inference-nim: ef70dad9934ed5c50478dcd772df08f0f0a41cc6c9a433edcdad5ce84fe92cf3 -rtx-pro-6000-eks-ubuntu-training-kubeflow: 1861f8389802b6592e667f217c45d965657c4f06875f98c53df6c906004280da -rtx-pro-6000-lke-ubuntu-inference: a876264218915286ebc002aa1134c5b64a25e9054f1fc88f59c9b68bcb80f2e0 -rtx-pro-6000-lke-ubuntu-training: a531171ba7c38877f1ade05c7cac7796d0907566ce7c6a3442502d51225d77b3 +a100-aks-ubuntu-training-kubeflow: 88974161cf87092faee1fd5ed39f6ef03ec217a630ee10ae34bee027dd8c97a4 +a100-any: 2cbbcd65973e54c164e8c0025142c43b0384e913c3ade2ebc4b3580e90270b17 +a100-eks-ubuntu-training-kubeflow: 66a2c9fc3172d79e1710a7522cca2c683b188193ddfc58d1b1953417b5354362 +a100-gke-cos-training-kubeflow: 34a990b268eac35a9fd00010563ef1bd964525bbbe83121b7aeeb54c7ba02954 +a100-oke-ubuntu-training-kubeflow: 5b79e3472e93ada835a7ec85b038857a1db3f4bf7df34033388b8c0c867a654e +b200-any: 96ce9531ff9d35b6f92577906e5c6a76a0df0648eb6ff0b338c7c92ae555b536 +b200-gke-cos-inference-dynamo: 9a257ac706f3f3b1955791441ed5b5b0c5acf17f0f1b113de40f1d74cd11df59 +b200-gke-cos-training-kubeflow: de66551f631b79451dd86ba10479748dcf7a911cc70ae538f3fdea5b0a8dd642 +bcm-inference: d7fe93b0a299ae87b15cf7fbb81a78b72b4d868a25c6fcbb70aa26f97a31f548 +gb200-any: e96732f58ce13d20e6b355afa20346bc25d88a9197d303064bdb5de91404279a +gb200-eks-ubuntu-inference-dynamo: 4a6f96ddb52b2e3b787ee0f65b6d4fd11052150f98ee17de4b4afe4401727619 +gb200-eks-ubuntu-training-kubeflow: b13b8d7ac8e45fedf00df2eb39699ddcdb5bd4c7bc94478c465aa582deb0b769 +gb200-eks-ubuntu-training-slurm: 03cac6712a2749ca6bb75f472544659c6a95d408bbddc56ac94aef5dc03c335a +gb200-oke-ubuntu-inference-dynamo: 9b771c100146719cd0a5534809dc4f0d70c84c19c62024fcae0aff20d355fa39 +gb200-oke-ubuntu-training-kubeflow: 905229e58cd9920e1b4ae57405518d482e3d098f1b01aa27509d9447cb6b2f55 +gb300-any: 51e94d57f8bf60b2d2920c8255a9933e2f1fcb8c8a594793c7666173154b03fe +gb300-eks-ubuntu-inference-dynamo: b61772b86011b7a39ba13d00dda699979e888846a00e72235a3f4bc573bb96f0 +gb300-eks-ubuntu-training-kubeflow: d187e3797e92b203c361518b3fbc5af76acd0a62071bcf3d3fa2c093cd94f027 +h100-aks-ubuntu-inference-dynamo: 9d4cb5e4cf170972b205aa483209d13ade1a94aea5b1f4c870f8c0146d3bfc2e +h100-aks-ubuntu-training-kubeflow: 30caffd8764966e7432672b1dcd74861012beaaf5ae0aeb32cd67ebca0c102af +h100-aks-ubuntu-training-slurm: b1cc76701977baab3aca432f0022f8c0842c547058cf3fbbad261db3b1c068b3 +h100-any: 63ec1b86ddeee974b2da15b1f8eafed9efab968a472b09d6179e1d117e667bc2 +h100-bcm-ubuntu-training: 657509ca168955fd6624efe6c7abab91040df4ac7b0aedf8a7ed4f5e7d00caa4 +h100-eks-ubuntu-inference-dynamo: b2db1bb4bb62dfa0790f9b77cb34b16f6261970b1420214f37b7ffa32bcfd5b5 +h100-eks-ubuntu-inference-nim: 0b3525d5cf7b471e27556d7b68f05e5b8c606d90a62cbf5d2b6dbeeb4aeff67d +h100-eks-ubuntu-training-kubeflow: f031ab2be281a5401cd3426fd5b8cb17020944938f38ddeaeb63540c96c149f1 +h100-eks-ubuntu-training-slurm: ff33f128fa254c9057c672a3c9b98452b7a648f20284ada0479446ca2b2b1f8f +h100-gke-cos-inference-dynamo: 141753509e44bba98e54d75eab96a0b720c987bd06ec8a4922ed42dfd4cc5006 +h100-gke-cos-training-kubeflow: f762c480f098f1f936ce7ab749f372fcfebf5d4ee11d06dbd473449bbb530e9a +h100-gke-cos-training-slurm: 28697b311b2d218e5bc88499720ae21f592bf751ec55b1f2975b9d7c37427b9f +h100-kind-inference-dynamo: 76360ba5eb316ce1814af34e812298e69004e1d82af09a9473ca9384f2126e72 +h100-kind-training-kubeflow: e77c4afe346b00d66a7329b5d74dc655c0219f15514dcbf58007c8db27564de3 +h100-kind-training-slurm: 76f993dd8d5e6dc113e38e98b01d99c1fa26157093e808345b09dfb5410a5059 +h200-any: d852f8915d79b5a8e527116f2aeede59df1d2d68969206484f40184afa8f5ce9 +h200-eks-inference: 479867958a103efcdb32b4592a2e1e1ca54f512ff855449ffbc8bd1e9dddb471 +h200-eks-training: 4078ec6949fe054f913665542b3256f103d8b5630855b61beea6568ac63a108a +l40-any: c386fac8b86684354177fb760c6cd1d41316d46d9a8438a1d817c82565aa013f +l40s-any: f493cca8f86fa263654bda25caeb840430f95b3713eac0b67426de1b20ec4f9b +l40s-oke-inference: 46d6020c6ce89505109ae2eb091789e58a5e5bbeb829c97e1a5e80c7d4d8302f +l40s-oke-training: 654cf1f167e7eebe6e6ddd4cd85a978f3170ad7c0c9a8a49bd853ad44990885c +monitoring-hpa: b86b773c621d5eca9fafc1e12750427111c0865a762496ce6aba28cf4ee4aa7a +ocp-inference-nim: bb3c7fa78c972241b47abedbff3069d4fad35ab0c64d5f0172c633d8ed610b27 +ocp-training: 1f457d2c2aed921c2a3281cca1a55b1181d979cc99f07c1b51b9ce827379b401 +rtx-pro-6000-any: ad8b509fb7f6c25c765d3b9112b03594e3aac99a48a9e6c11df790b131488284 +rtx-pro-6000-eks-ubuntu-inference-dynamo: 84cde160654f85a6e5fbe1f4f26f9d9c08efe388711678641040794f5b84ccb8 +rtx-pro-6000-eks-ubuntu-inference-nim: c99ee6060908f8edb35988b6a27e72f8189d55ee9365c48307335aee3557834e +rtx-pro-6000-eks-ubuntu-training-kubeflow: 07801d7883014186f8606926546b268c429c75245a4caa10cfdc6f006320e5da +rtx-pro-6000-lke-ubuntu-inference: e9d3717a2e093104fb3163a964150e4b44ec7f34a1958c76cc6e385c87e3f897 +rtx-pro-6000-lke-ubuntu-training: f09be1374361894e6cc7f60548b2a6003450405abb03c5a999e00132fbee2e69 diff --git a/pkg/bundler/validations/checks.go b/pkg/bundler/validations/checks.go index 22765a67a..2a2afe6fd 100644 --- a/pkg/bundler/validations/checks.go +++ b/pkg/bundler/validations/checks.go @@ -464,7 +464,7 @@ func driverAbsentRemedy(service recipe.CriteriaServiceType, os recipe.CriteriaOS "DaemonSet alongside it; see " + "docs/integrator/gke-gpu-setup.md." case recipe.CriteriaOSUbuntu: - // The pinned GPU Operator (v26.3.3) supports driver management + // The pinned GPU Operator supports driver management // on GKE only on Ubuntu node images with containerd. return "On GKE Ubuntu node images the GPU Operator can manage " + "the driver: bundle in GPU-Operator-managed mode: " + @@ -643,11 +643,11 @@ func effectiveComponentValues(ctx context.Context, recipeResult *recipe.RecipeRe // path.Clean'd (trailing-slash spellings compare equal, mirroring // pkg/recipe/driver_root_lockstep_test.go), declared empty string → // the default (the operator's own transformForDriverInstallDir treats -// "" identically to the default, gpu-operator v26.3.3). An explicitly +// "" identically to the default, gpu-operator v26.7.0). An explicitly // null or non-map hostPaths section is rejected with a blocking // message: Helm null-coalescing deletes a null key together with its // chart defaults, so the chart's unconditional .Values.hostPaths.rootFS -// access (clusterpolicy.yaml, v26.3.3) fails at install. A declared +// access (clusterpolicy.yaml, v26.7.0) fails at install. A declared // value that cleans to a relative path is rejected too — host-path // mounts require absolute paths. func resolveInstallDir(values map[string]any, componentName string) (string, bool, []string) { @@ -680,7 +680,7 @@ func resolveInstallDir(values map[string]any, componentName string) (string, boo // rejected rather than silently defaulted: the emitted values // would carry it verbatim, and the pinned ClusterPolicy CRD // types hostPaths.driverInstallDir as a string (gpu-operator - // v26.3.3 nvidia.com_clusterpolicies.yaml), so the install + // v26.7.0 nvidia.com_clusterpolicies.yaml), so the install // fails while a defaulted check would have validated against // /run/nvidia/driver instead. return installDir, false, []string{fmt.Sprintf( @@ -693,7 +693,7 @@ func resolveInstallDir(values map[string]any, componentName string) (string, boo if dir == "" { // Intentionally default-equivalent: the operator's own // transformForDriverInstallDir early-returns on "" exactly like - // the default (gpu-operator v26.3.3, controllers/object_controls.go). + // the default (gpu-operator v26.7.0, controllers/object_controls.go). return installDir, false, nil } cleaned := path.Clean(dir) @@ -716,7 +716,7 @@ func resolveInstallDir(values map[string]any, componentName string) (string, boo // resolveDRARoot resolves the effective nvidia-dra-driver-gpu // nvidiaDriverRoot for Rule 2. Only a genuinely ABSENT key falls back to -// the chart-default assumption ("/", DRA chart v0.4.1 values.yaml). A +// the chart-default assumption ("/", DRA chart v0.5.0 values.yaml). A // present null, empty-string, or non-string value is rejected: unlike the // gpu-operator's driverInstallDir (where "" is default-equivalent, see // resolveInstallDir), the DRA chart pipes the raw value through @@ -866,7 +866,7 @@ func nvsentinelDynamicGuardViolations(bundlerConfig *config.Config, componentNam // null-coalescing deletes the key together with its chart defaults, // so .Values.
is nil at render time and the gpu-operator // templates fail on unconditional field access (e.g. -// .Values.driver.manager.repository in _helpers.tpl, v26.3.3) — +// .Values.driver.manager.repository in _helpers.tpl, v26.7.0) — // ownership cannot be verified and the install would fail anyway. // A non-boolean toggle is rejected because the chart renders the // value unquoted, so YAML re-typing at install time can flip it to a @@ -1705,7 +1705,7 @@ func CheckNVSentinelDriverLabelDetectable(ctx context.Context, componentName str } // defaultRuntimeClassName is the shared chart default: the gpu-operator -// chart ships operator.runtimeClass: nvidia (v26.3.3, verified against +// chart ships operator.runtimeClass: nvidia (v26.7.0, verified against // the pinned chart values), and nvsentinel's metadata-collector subchart // ships runtimeClassName: "nvidia" (v1.9.0, charts/metadata-collector/ // values.yaml:31). Either side left unset therefore resolves to this diff --git a/pkg/bundler/validations/checks_test.go b/pkg/bundler/validations/checks_test.go index c72760c32..833f1718f 100644 --- a/pkg/bundler/validations/checks_test.go +++ b/pkg/bundler/validations/checks_test.go @@ -1330,7 +1330,7 @@ func TestCheckDriverOwnershipCoherence(t *testing.T) { // the key together with its chart defaults, so .Values.driver // is nil at render time and the chart's unconditional field // accesses (_helpers.tpl .Values.driver.manager.repository, - // v26.3.3) fail at install. Reject rather than default. The + // v26.7.0) fail at install. Reject rather than default. The // reachable vector is a top-level --set-json null: the typed // merge assigns it verbatim (mergeTypedValueByPath), while the // recipe-side overlay merge drops nil-valued keys before the @@ -1357,7 +1357,7 @@ func TestCheckDriverOwnershipCoherence(t *testing.T) { // hostPaths: null is the same hazard as a null driver/toolkit // section: Helm null-coalescing deletes the chart defaults and // clusterpolicy.yaml's unconditional .Values.hostPaths.rootFS - // access fails at install (v26.3.3). + // access fails at install (v26.7.0). name: "--set-json hostPaths=null → rejected", recipeResult: result("", aks, gpuOpRef(driverOn())), bundlerConfig: config.NewConfig(config.WithValueOverridesTypedPaths([]config.TypedComponentPath{ diff --git a/pkg/client/v1/gpu_driver_state.go b/pkg/client/v1/gpu_driver_state.go index dd4d2da7e..59801164d 100644 --- a/pkg/client/v1/gpu_driver_state.go +++ b/pkg/client/v1/gpu_driver_state.go @@ -109,7 +109,7 @@ func driverAbsentRemedy(service recipe.CriteriaServiceType, os recipe.CriteriaOS "DaemonSet and pins the driver version — nothing to deploy " + "by hand; see docs/integrator/gke-gpu-setup.md." case recipe.CriteriaOSUbuntu: - // The pinned GPU Operator (v26.3.3) supports driver management + // The pinned GPU Operator supports driver management // on GKE only on Ubuntu node images with containerd. return "On GKE Ubuntu node images the GPU Operator can manage " + "the driver: bundle in GPU-Operator-managed mode: " + diff --git a/pkg/recipe/computedomain_numnodes_test.go b/pkg/recipe/computedomain_numnodes_test.go new file mode 100644 index 000000000..799bd92cc --- /dev/null +++ b/pkg/recipe/computedomain_numnodes_test.go @@ -0,0 +1,248 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package recipe + +import ( + "io/fs" + "regexp" + "strings" + "testing" +) + +// specKeyRE matches the document's top-level `spec:` mapping key. +var specKeyRE = regexp.MustCompile(`^(\s*)spec\s*:\s*$`) + +// numNodesChildRE matches `numNodes:` at a given exact indentation. +func numNodesChildRE(indent string) *regexp.Regexp { + return regexp.MustCompile(`^` + regexp.QuoteMeta(indent) + `numNodes\s*:`) +} + +// specHasNumNodes reports whether a single YAML document declares numNodes as a +// DIRECT CHILD of spec. +// +// Path-aware on purpose. An earlier version matched `numNodes:` anywhere in the +// document, which accepted `metadata.numNodes` — a key Kubernetes ignores, while +// the required `spec.numNodes` stays absent and admission still fails. Matching +// the key without its parent is not a weaker check, it is the wrong check. +// +// Comment lines are stripped first: these manifests legitimately discuss +// "spec.numNodes: Required value" in prose, and a scan that does not strip them +// matches that instead of the real key, passing even when the key is deleted. +// +// A full YAML parse is unavailable — the manifests are Helm templates containing +// {{ }} expressions that no YAML parser accepts — so this walks indentation. +func specHasNumNodes(doc string) bool { + var lines []string + for _, line := range strings.Split(doc, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "#") || strings.TrimSpace(line) == "" { + continue + } + lines = append(lines, line) + } + for i, line := range lines { + m := specKeyRE.FindStringSubmatch(line) + if m == nil { + continue + } + specIndent := m[1] + var childRE *regexp.Regexp + for _, sub := range lines[i+1:] { + subIndent := sub[:len(sub)-len(strings.TrimLeft(sub, " \t"))] + // Dedent to spec's level or shallower ends the spec mapping. + if len(subIndent) <= len(specIndent) { + break + } + if childRE == nil { + childRE = numNodesChildRE(subIndent) + } + if childRE.MatchString(sub) { + return true + } + } + } + return false +} + +// computeDomainDocsMissingNumNodes returns the 0-based indexes of YAML +// documents that declare kind: ComputeDomain without spec.numNodes. +// +// Scoped per document: a multi-document manifest where one ComputeDomain sets +// the key and a second omits it would satisfy a whole-file scan while still +// failing admission. No such manifest exists in the catalog today; the guard is +// document-scoped so adding one cannot silently bypass it. +func computeDomainDocsMissingNumNodes(content string) []int { + var missing []int + for i, doc := range strings.Split(content, "\n---") { + if !strings.Contains(doc, "kind: ComputeDomain") { + continue + } + if !specHasNumNodes(doc) { + missing = append(missing, i) + } + } + return missing +} + +// TestComputeDomainManifestsSetNumNodes guards the fresh-install CRD-overlap +// hazard introduced by GPU Operator v26.7.0. +// +// Two charts in the catalog ship a CRD named computedomains.resource.nvidia.com: +// the standalone nvidia-dra-driver-gpu chart, and — new in gpu-operator +// v26.7.0 — the GPU Operator chart. The two copies are NOT identical. The +// operator's is a stale snapshot that lists numNodes as required and supplies +// no `default: 0`; the DRA driver's makes it optional with a default. +// +// Helm installs crds/ only when the CRD is absent and never upgrades it, and +// gpu-operator is ordered before nvidia-dra-driver-gpu. So on a FRESH cluster +// the operator's stricter copy is the one that lands, and any ComputeDomain CR +// omitting spec.numNodes is rejected by the API server with +// "spec.numNodes: Required value". Structural defaulting cannot rescue it +// because that copy carries no default. Neither chart installs a webhook that +// could supply the field. +// +// An UPGRADED cluster masks this: it already has the permissive copy installed +// by DRA 0.4.1, so the CR still admits. That asymmetry is why this is a unit +// guard rather than something an upgrade-path e2e would catch. +// +// The invariant: every ComputeDomain CR shipped in the catalog must set +// spec.numNodes explicitly, so it is valid under BOTH CRD copies regardless of +// which chart installed the CRD first. +// +// See PR #2439 and issue #1087 for the driver-root analog of this +// cross-component coupling problem. +func TestComputeDomainManifestsSetNumNodes(t *testing.T) { + t.Parallel() + + efs := GetEmbeddedFS() + + var checked int + err := fs.WalkDir(efs, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".yaml") { + return nil + } + raw, readErr := efs.ReadFile(path) + if readErr != nil { + return readErr + } + content := string(raw) + if !strings.Contains(content, "kind: ComputeDomain") { + return nil + } + checked++ + + for _, idx := range computeDomainDocsMissingNumNodes(content) { + t.Errorf("%s: YAML document %d declares kind: ComputeDomain but does not set spec.numNodes.\n"+ + " GPU Operator v26.7.0 ships a ComputeDomain CRD copy that marks numNodes\n"+ + " REQUIRED with no default, and it is installed before the DRA driver's\n"+ + " permissive copy. On a fresh cluster this CR is rejected at admission with\n"+ + " \"spec.numNodes: Required value\".\n"+ + " Set numNodes explicitly (0 is correct under IMEXDaemonsWithDNSNames=true,\n"+ + " the DRA driver default, where each IMEX daemon starts without waiting for\n"+ + " a quorum). See PR #2439.", path, idx) + } + return nil + }) + if err != nil { + t.Fatalf("walking embedded recipes: %v", err) + } + + // Fail closed on a vacuous pass: if the walk matched nothing, the guard is + // silently inert and a regression would go unnoticed. + if checked == 0 { + t.Fatal("no ComputeDomain manifests found in the embedded recipes — " + + "this guard is vacuous. Either the manifests moved, or the embed " + + "pattern no longer covers them.") + } + t.Logf("verified %d ComputeDomain manifest(s) set spec.numNodes", checked) +} + +// TestComputeDomainScannerCases pins the scanner's behavior directly, so the +// catalog guard above cannot quietly stop discriminating if the catalog changes. +// Each case is a shape that has either fooled a previous version of this +// scanner or must keep working. +func TestComputeDomainScannerCases(t *testing.T) { + t.Parallel() + + const header = "apiVersion: resource.nvidia.com/v1beta1\nkind: ComputeDomain\n" + + tests := []struct { + name string + doc string + wantMissing bool + }{ + { + name: "spec.numNodes present", + doc: header + "metadata:\n name: cd\nspec:\n numNodes: 0\n channel:\n allocationMode: All\n", + }, + { + name: "spec.numNodes absent", + doc: header + "metadata:\n name: cd\nspec:\n channel:\n allocationMode: All\n", + wantMissing: true, + }, + { + // Regression: an earlier scanner matched numNodes anywhere in the + // document, so this passed while admission would still fail. + name: "numNodes under metadata, not spec", + doc: header + "metadata:\n name: cd\n numNodes: 0\nspec:\n channel:\n allocationMode: All\n", + wantMissing: true, + }, + { + // Regression: an earlier scanner did not strip comments, so the + // prose in the real manifest satisfied it even with the key gone. + name: "numNodes only mentioned in a comment", + doc: header + "metadata:\n name: cd\nspec:\n # numNodes: Required value\n channel:\n allocationMode: All\n", + wantMissing: true, + }, + { + name: "nested numNodes does not satisfy the direct-child rule", + doc: header + "metadata:\n name: cd\nspec:\n channel:\n numNodes: 0\n", + // numNodes exists but under spec.channel, not spec. + wantMissing: true, + }, + { + name: "templated value is acceptable", + doc: header + "metadata:\n name: cd\nspec:\n numNodes: {{ .Values.numNodes }}\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := !specHasNumNodes(tt.doc) + if got != tt.wantMissing { + t.Errorf("specHasNumNodes reported missing=%v, want %v\ndoc:\n%s", + got, tt.wantMissing, tt.doc) + } + }) + } +} + +// TestComputeDomainMultiDocument covers the per-document scoping: a file where +// one ComputeDomain is valid and a second is not must report only the second. +func TestComputeDomainMultiDocument(t *testing.T) { + t.Parallel() + + content := "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: unrelated\n" + + "\n---\napiVersion: resource.nvidia.com/v1beta1\nkind: ComputeDomain\nmetadata:\n name: ok\nspec:\n numNodes: 0\n" + + "\n---\napiVersion: resource.nvidia.com/v1beta1\nkind: ComputeDomain\nmetadata:\n name: bad\nspec:\n channel:\n allocationMode: All\n" + + missing := computeDomainDocsMissingNumNodes(content) + if len(missing) != 1 || missing[0] != 2 { + t.Errorf("missing documents = %v, want [2] (only the third document lacks spec.numNodes)", missing) + } +} diff --git a/pkg/recipe/dra_k8s_floor_test.go b/pkg/recipe/dra_k8s_floor_test.go index 5baaf2056..c08bf8c55 100644 --- a/pkg/recipe/dra_k8s_floor_test.go +++ b/pkg/recipe/dra_k8s_floor_test.go @@ -52,9 +52,9 @@ var auditedDRAChartFloors = map[string]struct { minor int }{ // oci://registry.k8s.io/dra-driver-nvidia/charts/dra-driver-nvidia-gpu - // kubeVersion: ">=1.32.0-0" - "nvidia-dra-driver-gpu": {version: "0.4.1", minor: 32}, - "nvidia-dra-driver-gpu-ocp": {version: "0.4.1", minor: 32}, + // kubeVersion: ">=1.32.0-0" (unchanged from 0.4.1 to 0.5.0) + "nvidia-dra-driver-gpu": {version: "0.5.0", minor: 32}, + "nvidia-dra-driver-gpu-ocp": {version: "0.5.0", minor: 32}, } // draChartKubeVersionMinor is the highest audited floor across the enrolled DRA diff --git a/pkg/recipe/driver_root_lockstep_test.go b/pkg/recipe/driver_root_lockstep_test.go index 1372e6141..e1c83fdc7 100644 --- a/pkg/recipe/driver_root_lockstep_test.go +++ b/pkg/recipe/driver_root_lockstep_test.go @@ -74,8 +74,8 @@ import ( // **Why "explicitly set" matters for the lockstep case.** An empty value // falls through to the upstream chart's bundled default, which the test // cannot read — and per-component defaults differ (GPU Operator chart -// 26.3.3 defaults driverInstallDir to /run/nvidia/driver, but DRA chart -// 25.12.0 defaults nvidiaDriverRoot to /). Relying on chart defaults is +// 26.7.0 defaults driverInstallDir to /run/nvidia/driver, but DRA chart +// 0.5.0 defaults nvidiaDriverRoot to /). Relying on chart defaults is // itself drift waiting to happen on the next chart bump, so when the // lockstep applies the test treats "not explicitly set on both" as a // failure. @@ -211,7 +211,7 @@ func TestDriverRootLockstep(t *testing.T) { t.Errorf( "overlay %q: both nvidia-dra-driver-gpu.nvidiaDriverRoot and gpu-operator.hostPaths.driverInstallDir are unset.\n"+ " Both must be set explicitly to the same path. Chart defaults differ across components\n"+ - " (gpu-operator chart 26.3.3: /run/nvidia/driver; dra chart 25.12.0: /), so an unset value\n"+ + " (gpu-operator chart 26.7.0: /run/nvidia/driver; dra chart 0.5.0: /), so an unset value\n"+ " is drift waiting to happen on the next chart bump.\n"+ " See issue #1087.", name) diff --git a/pkg/recipe/testdata/catalog_parity_golden.yaml b/pkg/recipe/testdata/catalog_parity_golden.yaml index 1250f0aa0..2e8b4acd7 100644 --- a/pkg/recipe/testdata/catalog_parity_golden.yaml +++ b/pkg/recipe/testdata/catalog_parity_golden.yaml @@ -3,52 +3,52 @@ # # One entry per leaf overlay: sha256 of its deterministically-marshalled # resolved recipe. A moved digest means that recipe's resolved bytes changed. -a100-aks-ubuntu-training-kubeflow: 47f2c604ad1491346fc4923dacafe6d636fce9fc62b7312e1972f2286e7a1184 -a100-any: 5abf30208c87a34a0f7a8ace6fb04b878de14233bb8691fe7344f1500985eed3 -a100-eks-ubuntu-training-kubeflow: 3cda91f7acec765b159ce307509479e30a9035acff615da06eab3a0b5d6d24c4 -a100-gke-cos-training-kubeflow: 7632830f5d0b5b669f6f4d509233c4314efc29f57152230d376fb2473eca7186 -a100-oke-ubuntu-training-kubeflow: 6619d20510945e05270ede31d1124313d45959f9cdfb189d63e5cd0f9eea7b16 -b200-any: c38ec9744d305fe7de0a5a590d445f028e38b0047786fe80c6a3c349ea95f02b -b200-gke-cos-inference-dynamo: a04ec4b0c5e24a12e08d29a7ed6f6cd8ff23cc53665fdca3f2ff89658507664d -b200-gke-cos-training-kubeflow: 62eec84f2f80c6f08fbfb2dadbd859e2ba07e787e43b58c5bdcc8360cf4aee71 -bcm-inference: 799569869ffff42841c5e758d6da1bc6f474af41a3cca5f74f806597e5be2f10 -gb200-any: 0dfa723aa4cc4850bea174f575e2853d4760c56035efb4acffa4d69b3ad2b886 -gb200-eks-ubuntu-inference-dynamo: c48c3c77fbc34e2f57223b97e5dc1fd2150fee3fae4a7ecbeae46394180f0c48 -gb200-eks-ubuntu-training-kubeflow: 36468d7c502b13e3ae6e441eecd0c6da7901fc2c89dfeac014d5df184cd74700 -gb200-eks-ubuntu-training-slurm: c660bbe8b1a97e250cdc7a61e75ea58be49dc96061626715f516bcbf885f2d13 -gb200-oke-ubuntu-inference-dynamo: 4ee02af889b1563108700d00575d418f0bd55f716404086579e16ece32700fe0 -gb200-oke-ubuntu-training-kubeflow: a2828e176878a36533d39a6ba720418bb6cb3eff34a6de7b2e7db83197248d69 -gb300-any: 4d22bb7186907f546733386a08efb32c8824552890b2e8d2c90cc717d201fb0d -gb300-eks-ubuntu-inference-dynamo: 5be01d4581d333a84a8597a77c192159a8bafe93104d6441ff8a0b6ad562c9c1 -gb300-eks-ubuntu-training-kubeflow: 33b8adf01bc622198ba9d46731c23efb88ab44991b84c8e37eea4ddf19aaeea9 -h100-aks-ubuntu-inference-dynamo: 46e0a581b1169cc1e5a4d3226aa46f4eb1a2fdad23ab37fba5c4825d773e80e5 -h100-aks-ubuntu-training-kubeflow: 3be5d233181d6dea5bf7127f554be827c579bac2fc893defca81621bbcc7ee77 -h100-aks-ubuntu-training-slurm: 2d797149b58ce853e7150d908be331c61dfdb477220fae9c15f6ebea22f9fe22 -h100-any: 2896754239e04cf8490a29c1a86b9a7d543cf982eb557c363f1e586668c1fe12 -h100-bcm-ubuntu-training: 85c125e8440059a5087e3835551edb0196bf9256f27f84cb4fa094b3517536e9 -h100-eks-ubuntu-inference-dynamo: 1a3f60c84c015754794edca31eb5abcef6444ed155e3470aca4bab4be3a7db3d -h100-eks-ubuntu-inference-nim: 53a96e375f75eed6508e57978e1e07fd05f740688dc4ca5a0e2fd04b90d716f3 -h100-eks-ubuntu-training-kubeflow: ecac42280c08d1eccaa49c31dee97009c803c360465d942de68a4d9786e8334c -h100-eks-ubuntu-training-slurm: 5fc03eb17bbb1934a78c03576d4aeeda8e723ca0937ca2fb0f6e0e40c19d38f2 -h100-gke-cos-inference-dynamo: 0c3d27435ece2300c0e88855f362d4e4cded31b9006c3d641560cab3fdffb2fd -h100-gke-cos-training-kubeflow: a6b36e216007c0be87bfadd22012ab0c25a94ce25a9a27e4d1c190bbaf63adb3 -h100-gke-cos-training-slurm: a4aaadffe6e17e6d01f2bdea68e5ea2d76fff2fc364b27ed811d9cf477763646 -h100-kind-inference-dynamo: 805b8224a75834e82d34cc8228e81a16e96a67b6b5742c9a329fc707c6deb6c8 -h100-kind-training-kubeflow: 5dd6d4efb2585c41c58dba1f373a5738c19be44618d109d5e31a53cf56482bd8 -h100-kind-training-slurm: 355c088589f5c948e025483d71468509addec60544b137653bf4df07f222ccc1 -h200-any: 15d07296d4050a66f0df218d3a41435541f69bb965c643869bc775e41988b4b5 -h200-eks-inference: 68b97d51d0cbc982564175114cb8626824e7643ac2fb6babf62073311c7eeb60 -h200-eks-training: 6db44e06178a0f74fab4d554406f988fe2e6c119b9523e458eb470ab3ed0cc8d -l40-any: abcbb63d3cd95efac239582f156b5af2c116079e3c48407fd275c889d0b2ad12 -l40s-any: 1bd1432c216e2e9f7b9ef061916e12e667aae1eb98fb3311857e2bc5c6dee44e -l40s-oke-inference: 0e0ef1669b6cb89ff3596a4b4c66f809b607d13ab264b1a02b82ad797c61b2bb -l40s-oke-training: bc8f2c4b08609739102969c4cc610045073834024b57b8121d2a715dbe8768c2 -monitoring-hpa: ca01840738fc06b47eb6713d630304ac2ca40693ed6bfbac9c57f7c8da4294a4 -ocp-inference-nim: fe0fc043ef8da714a0b3ff20b589faeba23a1bf7507380c3cf4c3f8ea6c37987 -ocp-training: 7c5beb2c2e001147b01e507a3d7e9cb1b2fa8c1cc20d6f86b02ebb271c00e390 -rtx-pro-6000-any: 945f6d4425ee52df4491fc74da78184a4d5f04f0204db4950694486993cc9244 -rtx-pro-6000-eks-ubuntu-inference-dynamo: 388abe59dc2fa07359455d387364e2e9fc3ed3922a9d0241604f67c8b65a1913 -rtx-pro-6000-eks-ubuntu-inference-nim: db4c9c06a32f435d04160a5e01198af6e6b42acbc90c7eb681378aa887cc3313 -rtx-pro-6000-eks-ubuntu-training-kubeflow: 5a1423754da34608921d125285e22e2ceaa053f494ae27b97892f1133e4a3f47 -rtx-pro-6000-lke-ubuntu-inference: c2a6382a95758cc36ebc77d8b157b2aecbdd9386849ac5153b94e0d657baa6dc -rtx-pro-6000-lke-ubuntu-training: 1fad1edc2ef174836e20cd4e4d4e64ff350d6ba4d57711e095318fee978cd398 +a100-aks-ubuntu-training-kubeflow: 1cc3ce4d52a71b6137179afa51859e149b656e71c32c89c8be23f193f7d28ed2 +a100-any: 5becb0645a3c787027e459adbcd093c8656f6c1e9370c90f9827e4978bca580c +a100-eks-ubuntu-training-kubeflow: a743c24bf62271a407b6608dd6099113ce4baaabe9506b016df6a3088e0c80c7 +a100-gke-cos-training-kubeflow: a8f9afa0cf6045d7789d1385b5cdabe654170da00a3894de446c34100dd60acb +a100-oke-ubuntu-training-kubeflow: 2bc755fc494c4db63788544b7bad2c98dfc0b4322a1cbe65364993fb40e38db6 +b200-any: 74665d86b2781705d2ecc7e28a7c45ecac931560095bacdd48cfcc5249714ac0 +b200-gke-cos-inference-dynamo: de879f8526521e4c5198cee2c91f29b5ed04ec8ab64a9e28def821341105c93d +b200-gke-cos-training-kubeflow: 5d8da216a34ed3935c2b0f0732f3bcf1187081585bce4f28610330a03e7b7ab6 +bcm-inference: fbad7e9704b9fbcd099ce9ea0799cecb0a0eb0fbc46f8b758ea5c1299be71db4 +gb200-any: 55cd7d2a2d33f677f10d58a1ddfbd02b79026e7a959d19d244c229437f8d8b69 +gb200-eks-ubuntu-inference-dynamo: 8aeaa6b60e36f20e3321c14160d3c122f3583937ed277bfff499ca4acbb691a2 +gb200-eks-ubuntu-training-kubeflow: 8510ee4d66330a42add297565d531c8daa5d525006b0afa1e07dd442c72c8f36 +gb200-eks-ubuntu-training-slurm: ecafb0d201f905a1a09556394cec2bdc601de3c556e9c7502bc98d998d3fe17e +gb200-oke-ubuntu-inference-dynamo: 6ee3c58c8363a91d84334817fddef175e0bde63aee6f710b467a941953f6fc4d +gb200-oke-ubuntu-training-kubeflow: 21b58a301871cffdd40de2c10fd4f6ec55833a9621a5eebf864816ed941d8adb +gb300-any: e46a40fd6441b6e1348d8b9bcb60ef78623e22980f2af75b466b7aaa2b6a9960 +gb300-eks-ubuntu-inference-dynamo: 414ea9efccad9f9b1f7bfdb738829f0c9310ba3621bae6d9da18ae6cce8137ad +gb300-eks-ubuntu-training-kubeflow: 9e04ac26502c5e3968a08257062ac01fd53d28962c408b72e68f6a95db22ea8f +h100-aks-ubuntu-inference-dynamo: d941fc63190422765c02db2f5f017da7131adc687ca1e25dfa2b53b120117787 +h100-aks-ubuntu-training-kubeflow: a8ad56b9c749ec8cc84d3c866ed8520cf27bc364a067938101ad635633071dc2 +h100-aks-ubuntu-training-slurm: 9791558c8e0e44b8462511d5ef03fa21c26bf0a3993fe17d11c4d9e8c4420605 +h100-any: 887124130cfc77b01a0c3dc8385782de13b4ad359314e607a6034443f803884c +h100-bcm-ubuntu-training: 790117ef6e991a27720954877e6f9e0e6d54b6f32c3988a22c388c104663c8ae +h100-eks-ubuntu-inference-dynamo: 9ecd22288407079e2255adbe353c25fbf441792f827baf7066c3df5cd50eb68e +h100-eks-ubuntu-inference-nim: 6db15a210537c5695abbf7ebb57c2cfc33c4cab97ce9b0007d366bdc1bda33e6 +h100-eks-ubuntu-training-kubeflow: 677de790e9697c102e850c2394d521fdba4ba6853004f196c65f59abba051ea6 +h100-eks-ubuntu-training-slurm: 85ad4d6705e0ea7bc73d91fb9a1e99e5026adbe9dc6a322ac0bb3187fbb1ac78 +h100-gke-cos-inference-dynamo: 9f6f269901e88eb29b00b3bdcfeba8ee7e39d3b5ed654257b0d3e2334918cc2d +h100-gke-cos-training-kubeflow: bc98f3822cca567c496a91441919089f0d260a2f8cb8fedf6c678a89bf03cfb7 +h100-gke-cos-training-slurm: f1aad8397ac49789794875ca0836189be018c120c98ac718edf9f5fd90356b0c +h100-kind-inference-dynamo: 1c4cf3a98057275dadd724b0f068f32c4e9a8ae8744569d3a2814e7336938344 +h100-kind-training-kubeflow: b5dfa79c42619c5b5995f932af56f343e4d0588344e8d37d36cab7483d1cee03 +h100-kind-training-slurm: 8549b91c859978283d7371666565273a1c75e2908d9167ccaafa75f019b4d6de +h200-any: d8cce8f74339053cb6c3ce33d99e12012f73b8c675a77403b42d6e68d2a5d53d +h200-eks-inference: 0232aa2a3399cfd7faf04950fdb0136df4174c79a5999814a4dc5f8b57ac0893 +h200-eks-training: 5ca88ad1c13f778513cc972f28ff38736d266e6187d87d2d30ece8e6013353fe +l40-any: 9d5abac3c21c8bca2e4d8b2015691dc3f4f834db4590643c7a5a55ac0398fc35 +l40s-any: 79a8c1afe261c6e214e1f3019c215625c588192a0af917e6c5ab0049edd7d4df +l40s-oke-inference: 3c8d3fb233f8ee8d608c4731c5dad38d9237cc04960b2e46051313062a1ad267 +l40s-oke-training: 58c317707e06fd78ff328f4f420fba5963b1e89962907717b10d1da2980ba454 +monitoring-hpa: 8206ceb11958dadac846292bb693e7757762528d99d00316e4aa427ec4cb3ea9 +ocp-inference-nim: f1f99e6e99b4f7084ab4cba65e38dddbefb9d7bcf9579f16c86d5f76508b232f +ocp-training: 4c5d4e277307f9f08c7ed5a47d79e61b7a8bbc5970ba6554a0a4471b4e7d405e +rtx-pro-6000-any: c54af648defc635cb8540064f4eb1cf605e4a95dd3f221a089cdfcc29d409438 +rtx-pro-6000-eks-ubuntu-inference-dynamo: 9012a9b9108e1f76dc129bdc2407bef55ef1b815dd93e3a484dc9ff8dfd6b7ba +rtx-pro-6000-eks-ubuntu-inference-nim: b226e782c924bfabec2b20a9533c6eac575cb9ad310a7e9cc6e0d28202e15a49 +rtx-pro-6000-eks-ubuntu-training-kubeflow: b567ecb933095717694b982d0650a349dd5f9929eb6c3d39a1b0e1d4fba73772 +rtx-pro-6000-lke-ubuntu-inference: aeb331c840ec81628163a7056ffed7295ffd9d57b05fa3b8b0ac2e96f36baf51 +rtx-pro-6000-lke-ubuntu-training: 2ddcfbd4a02c0ca4cce94187fe3ce8b421cd5436e25a1d387a6666c475cb169a diff --git a/recipes/components/gpu-operator-ocp/values.yaml b/recipes/components/gpu-operator-ocp/values.yaml index 1f090147d..8b931edbc 100644 --- a/recipes/components/gpu-operator-ocp/values.yaml +++ b/recipes/components/gpu-operator-ocp/values.yaml @@ -160,7 +160,7 @@ daemonsets: # operator.upgradeCRD: Helm chart setting, not a ClusterPolicy field. # operator.resources: Helm chart setting for the operator Deployment, not CR. # driver.version: OCP operator manages driver version via the certified driver container. -# driver.useOpenKernelModules: OCP uses pre-built driver containers from the certified catalog. +# driver.kernelModuleType: OCP uses pre-built driver containers from the certified catalog. # driver.maxParallelUpgrades: Use daemonsets.rollingUpdate.maxUnavailable instead. # devicePlugin.env: OCP sets device plugin env via the operator's own defaults. # validator: OCP operator manages validation internally. diff --git a/recipes/components/gpu-operator/manifests/nvidia-toolkit-hardening-aks.yaml b/recipes/components/gpu-operator/manifests/nvidia-toolkit-hardening-aks.yaml index f8a440c85..885dbc00a 100644 --- a/recipes/components/gpu-operator/manifests/nvidia-toolkit-hardening-aks.yaml +++ b/recipes/components/gpu-operator/manifests/nvidia-toolkit-hardening-aks.yaml @@ -55,7 +55,7 @@ # device-request path open — the same posture as GPU-Operator-managed mode, # not a regression — so full multi-tenant isolation additionally requires # an admission policy restricting a /dev/null-backed mount whose destination -# is under /var/run/nvidia-container-devices (the pinned toolkit v1.19.1 +# is under /var/run/nvidia-container-devices (the pinned toolkit v1.20.0 # accepts the volume-mount device request only when the mount SOURCE is # /dev/null). secure-accelerator-access does not # exercise the volume-mounts path, so a green result is not proof of diff --git a/recipes/components/gpu-operator/values.yaml b/recipes/components/gpu-operator/values.yaml index a107a5943..569d102d2 100644 --- a/recipes/components/gpu-operator/values.yaml +++ b/recipes/components/gpu-operator/values.yaml @@ -147,16 +147,60 @@ gfd: enabled: true driver: - # NVIDIA's recommended driver for the v26.3.3 chart; matches the - # GB200+EFA floor so a single global pin covers H100/B200/GB200 EKS. + # HELD at 580.173.02 — deliberately BEHIND the v26.7.0 chart default of + # 595.91.07. Do not advance without EFA validation on P6e hardware. + # + # NVIDIA documents R595 as backward incompatible with P6e-GB200 EFA. R595 + # drops NVreg_GrdmaPciTopoCheckOverride (the R580-era escape hatch) and + # substitutes a topology check: EFA must sit in an IOMMU group, or GPU and + # EFA must share a root port. p6e-gb200/gb300 satisfies neither — measured + # on p6e-gb300r.36xlarge, where EFA (0000:97:00.0) has no iommu_group and + # sits under pci0000:84 while the GPU (0000:9d:00.0) is in group 0 under + # pci0000:98. + # + # The failure is proven, not inferred. Running the NCCL NET workload under + # 595 with the preflight bypassed, both nodes selected aws-ofi-nccl/EFA/RDMA + # and then aborted, with the kernel logging: + # NVRM: dma-buf attach failed: topology not supported for mapping type FORCE_PCIE + # Under 580.173.02 on the same two nodes, NCCL NET passed at 43.16 GB/s. + # + # Scope: the AWS P6e Grace-Blackwell + EFA GPUDirect path specifically. NVLS + # passed under 595 (840 GB/s), as did deployment and conformance; NVLink + # never touches EFA. + # + # Holding here is a supported combination, not an untested one: GPU Operator + # v26.7.0 lists 580.173.02 as supported (595.91.07 is merely its default), + # and DRA driver 0.5.0 requires >= 580. Pinned rather than inherited so a + # chart bump is an explicit driver decision, and so BOM/attestation inputs + # stay deterministic. + # + # Do not substitute a newer R580 patch without re-validating EFA — + # 580.173.02 is the exact measured baseline. Per-family overrides, so + # non-P6e families can advance independently, are tracked separately. version: 580.173.02 enabled: true - useOpenKernelModules: true + # Replaces useOpenKernelModules, which the chart deprecated and made a + # no-op (identical NOTE in v26.3.3 and v26.7.0 values.yaml), so the old + # key had already stopped having any effect before this pin moved. + # + # "auto" pins today's effective behaviour, not a change: it is the chart + # default both before and after this bump, and it is what the OCP + # ClusterPolicy template already falls back to + # (components/gpu-operator-ocp/manifests/clusterpolicy.yaml). Per the + # ClusterPolicy CRD, auto selects the recommended module type from the + # host's GPU devices and driver branch — which yields open modules on + # Blackwell/Grace, where they are required. + # + # Setting "open" instead would pin the intent the dead key expressed, but + # that IS a behaviour change on any hardware where auto currently selects + # proprietary, so it needs its own validation rather than riding a version + # bump. + kernelModuleType: auto maxParallelUpgrades: 5 rdma: # Default off: nvidia_peermem only loads against Mellanox MOFED # symbols. AWS EFA (EKS p4d/p5/p5e) and Linode have no MOFED, so - # peermem fails to load and v26.3.3's stricter driver-validation + # peermem fails to load and the chart's stricter driver-validation # init container blocks the rest of the GPU stack. Overlays that # ship MOFED (AKS via network-operator) explicitly re-enable this. enabled: false @@ -189,8 +233,9 @@ validator: nfd: enabled: false -# Confidential Compute Manager defaults to enabled in chart v26.3.x; keep -# it off until AICR has explicit CC-capable hardware support. +# Confidential Compute Manager defaults to enabled in the upstream chart +# (verified still true in v26.7.0); keep it off until AICR has explicit +# CC-capable hardware support. ccManager: enabled: false diff --git a/recipes/components/nvidia-dra-driver-gpu-ocp/manifests/scc-rolebinding.yaml b/recipes/components/nvidia-dra-driver-gpu-ocp/manifests/scc-rolebinding.yaml index a04352c57..d0ecd406e 100644 --- a/recipes/components/nvidia-dra-driver-gpu-ocp/manifests/scc-rolebinding.yaml +++ b/recipes/components/nvidia-dra-driver-gpu-ocp/manifests/scc-rolebinding.yaml @@ -26,7 +26,7 @@ # SCC (see CodeRabbit CWE-732 finding on PR #2057). Name confirmed via: # helm template nvidia-dra-driver-gpu-ocp \ # oci://registry.k8s.io/dra-driver-nvidia/charts/dra-driver-nvidia-gpu \ -# --version 0.4.1 -f values.yaml --namespace nvidia-dra-driver +# --version 0.5.0 -f values.yaml --namespace nvidia-dra-driver apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/recipes/components/slinky-slurm/manifests/compute-domain.yaml b/recipes/components/slinky-slurm/manifests/compute-domain.yaml index 3bf024787..3af09f839 100644 --- a/recipes/components/slinky-slurm/manifests/compute-domain.yaml +++ b/recipes/components/slinky-slurm/manifests/compute-domain.yaml @@ -29,6 +29,21 @@ metadata: name: slinky-slurm-imex namespace: {{ .Release.Namespace }} spec: + # Pinned explicitly rather than relying on the CRD default. GPU Operator + # v26.7.0 ships its own copy of resource.nvidia.com_computedomains.yaml in + # crds/, and that copy is a stale snapshot: it lists numNodes as REQUIRED + # and carries no `default: 0`, where the DRA driver chart's copy makes it + # optional with a default. Helm installs crds/ only when the CRD is absent + # and never upgrades it, and gpu-operator is ordered before + # nvidia-dra-driver-gpu, so on a fresh cluster the operator's stricter copy + # wins and a CR omitting numNodes is rejected with "spec.numNodes: Required + # value". Setting it here is valid under both copies. + # + # 0 is the correct value, not merely a placeholder: with + # IMEXDaemonsWithDNSNames=true (the DRA driver default) each IMEX daemon + # starts immediately instead of waiting for a quorum. Matches + # buildComputeDomain() in validators/performance/nccl_all_reduce_bw_constraint.go. + numNodes: 0 channel: allocationMode: All resourceClaimTemplate: diff --git a/recipes/overlays/ocp.yaml b/recipes/overlays/ocp.yaml index 02c0e7c93..3fffe60cb 100644 --- a/recipes/overlays/ocp.yaml +++ b/recipes/overlays/ocp.yaml @@ -22,7 +22,7 @@ spec: service: ocp constraints: - # nvidia-dra-driver-gpu-ocp (chart 0.4.1) declares kubeVersion: + # nvidia-dra-driver-gpu-ocp (chart 0.5.0) declares kubeVersion: # '>=1.32.0-0'. Raised from >= 1.29 so DRA's floor doesn't silently # pass recipe-time constraint checks and then fail at `helm install`. # This effectively requires OCP 4.19+ (first OCP minor shipping k8s 1.32). diff --git a/recipes/registry.yaml b/recipes/registry.yaml index 7073151e1..15cc959f9 100644 --- a/recipes/registry.yaml +++ b/recipes/registry.yaml @@ -100,7 +100,7 @@ components: helm: defaultRepository: https://helm.ngc.nvidia.com/nvidia defaultChart: nvidia/gpu-operator - defaultVersion: v26.3.3 + defaultVersion: v26.7.0 defaultNamespace: gpu-operator nodeScheduling: system: @@ -396,7 +396,7 @@ components: helm: defaultRepository: oci://registry.k8s.io/dra-driver-nvidia/charts defaultChart: dra-driver-nvidia-gpu - defaultVersion: "0.4.1" + defaultVersion: "0.5.0" defaultNamespace: nvidia-dra-driver nodeScheduling: system: @@ -1102,7 +1102,7 @@ components: helm: defaultRepository: oci://registry.k8s.io/dra-driver-nvidia/charts defaultChart: dra-driver-nvidia-gpu - defaultVersion: "0.4.1" + defaultVersion: "0.5.0" defaultNamespace: nvidia-dra-driver nodeScheduling: system: diff --git a/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml b/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml index 5b59ffdac..c38d7c7d3 100644 --- a/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml +++ b/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml @@ -39,14 +39,14 @@ daemonsets: # ── Driver: nvidia_peermem off on EKS (AWS EFA path uses aws-ofi-nccl) # nvidia_peermem only loads against Mellanox MOFED symbols; on AWS EFA -# (p4d/p5/p5e) it fails to load and v26.3.3's strict driver-validation +# (p4d/p5/p5e) it fails to load and the chart's strict driver-validation # init container blocks the rest of the GPU stack. NCCL multi-node on # EFA uses libfabric via aws-ofi-nccl, not nvidia_peermem. driver: enabled: true rdma: enabled: false - useOpenKernelModules: true + kernelModuleType: auto # ── GDRCopy: GPU-direct memory for high-performance training ───────── gdrcopy: