From 220b1bc924d83001c744f56423cd0300a28816eb Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 09:28:15 -0700 Subject: [PATCH 01/16] chore(recipes): bump gpu-operator to v26.7.0, driver to 595.91.07, DRA to 0.5.0 Signed-off-by: Yuan Chen --- recipes/components/gpu-operator/values.yaml | 12 ++++++++---- .../manifests/scc-rolebinding.yaml | 2 +- recipes/overlays/ocp.yaml | 2 +- recipes/registry.yaml | 6 +++--- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/recipes/components/gpu-operator/values.yaml b/recipes/components/gpu-operator/values.yaml index a107a5943..de4866fd3 100644 --- a/recipes/components/gpu-operator/values.yaml +++ b/recipes/components/gpu-operator/values.yaml @@ -147,16 +147,20 @@ gfd: enabled: true driver: - # NVIDIA's recommended driver for the v26.3.3 chart; matches the - # GB200+EFA floor so a single global pin covers H100/B200/GB200 EKS. - version: 580.173.02 + # NVIDIA's recommended driver for the v26.7.0 chart; a single global + # pin covers H100/B200/GB200 EKS. Pinned rather than inherited from the + # chart default so a future chart bump is an explicit driver decision + # instead of a silent fleet-wide driver change, and so BOM/attestation + # inputs stay deterministic. Moves with the chart: holding an older + # driver against a newer chart is a combination upstream does not test. + version: 595.91.07 enabled: true useOpenKernelModules: true maxParallelUpgrades: 5 rdma: # Default off: nvidia_peermem only loads against Mellanox MOFED # symbols. AWS EFA (EKS p4d/p5/p5e) and Linode have no MOFED, so - # peermem fails to load and v26.3.3's stricter driver-validation + # peermem fails to load and the chart's stricter driver-validation # init container blocks the rest of the GPU stack. Overlays that # ship MOFED (AKS via network-operator) explicitly re-enable this. enabled: false diff --git a/recipes/components/nvidia-dra-driver-gpu-ocp/manifests/scc-rolebinding.yaml b/recipes/components/nvidia-dra-driver-gpu-ocp/manifests/scc-rolebinding.yaml index a04352c57..d0ecd406e 100644 --- a/recipes/components/nvidia-dra-driver-gpu-ocp/manifests/scc-rolebinding.yaml +++ b/recipes/components/nvidia-dra-driver-gpu-ocp/manifests/scc-rolebinding.yaml @@ -26,7 +26,7 @@ # SCC (see CodeRabbit CWE-732 finding on PR #2057). Name confirmed via: # helm template nvidia-dra-driver-gpu-ocp \ # oci://registry.k8s.io/dra-driver-nvidia/charts/dra-driver-nvidia-gpu \ -# --version 0.4.1 -f values.yaml --namespace nvidia-dra-driver +# --version 0.5.0 -f values.yaml --namespace nvidia-dra-driver apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/recipes/overlays/ocp.yaml b/recipes/overlays/ocp.yaml index 02c0e7c93..3fffe60cb 100644 --- a/recipes/overlays/ocp.yaml +++ b/recipes/overlays/ocp.yaml @@ -22,7 +22,7 @@ spec: service: ocp constraints: - # nvidia-dra-driver-gpu-ocp (chart 0.4.1) declares kubeVersion: + # nvidia-dra-driver-gpu-ocp (chart 0.5.0) declares kubeVersion: # '>=1.32.0-0'. Raised from >= 1.29 so DRA's floor doesn't silently # pass recipe-time constraint checks and then fail at `helm install`. # This effectively requires OCP 4.19+ (first OCP minor shipping k8s 1.32). diff --git a/recipes/registry.yaml b/recipes/registry.yaml index 7073151e1..15cc959f9 100644 --- a/recipes/registry.yaml +++ b/recipes/registry.yaml @@ -100,7 +100,7 @@ components: helm: defaultRepository: https://helm.ngc.nvidia.com/nvidia defaultChart: nvidia/gpu-operator - defaultVersion: v26.3.3 + defaultVersion: v26.7.0 defaultNamespace: gpu-operator nodeScheduling: system: @@ -396,7 +396,7 @@ components: helm: defaultRepository: oci://registry.k8s.io/dra-driver-nvidia/charts defaultChart: dra-driver-nvidia-gpu - defaultVersion: "0.4.1" + defaultVersion: "0.5.0" defaultNamespace: nvidia-dra-driver nodeScheduling: system: @@ -1102,7 +1102,7 @@ components: helm: defaultRepository: oci://registry.k8s.io/dra-driver-nvidia/charts defaultChart: dra-driver-nvidia-gpu - defaultVersion: "0.4.1" + defaultVersion: "0.5.0" defaultNamespace: nvidia-dra-driver nodeScheduling: system: From b306ce2a253929b43b4ebe438ca24b69d0686ef8 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 09:30:11 -0700 Subject: [PATCH 02/16] docs: regenerate container-images BOM for the gpu-operator and DRA bump Signed-off-by: Yuan Chen --- docs/user/container-images.md | 38 +++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/user/container-images.md b/docs/user/container-images.md index a617f14b9..dcf0735f1 100644 --- a/docs/user/container-images.md +++ b/docs/user/container-images.md @@ -42,7 +42,7 @@ _Rendering fidelity:_ `catalog-parity: charts are rendered with the shared recip | gatekeeper | helm | gatekeeper/gatekeeper | 3.22.2 | 3 | | gcp-driver-installer | manifest | — | — | 3 | | gke-nccl-tcpxo | manifest | — | — | 4 | -| gpu-operator | helm | nvidia/gpu-operator | v26.3.3 | 15 | +| gpu-operator | helm | nvidia/gpu-operator | v26.7.0 | 15 | | gpu-operator-ocp | manifest | — | — | 0 | | gpu-operator-ocp-olm | manifest | — | — | 0 | | grove | helm | grove-charts | v0.1.0-alpha.12 | 1 | @@ -64,8 +64,8 @@ _Rendering fidelity:_ `catalog-parity: charts are rendered with the shared recip | nfd-ocp-olm | manifest | — | — | 0 | | nodewright-customizations | manifest | — | — | 5 | | nodewright-operator | helm | nodewright | v0.17.1 | 3 | -| nvidia-dra-driver-gpu | helm | dra-driver-nvidia-gpu | 0.4.1 | 1 | -| nvidia-dra-driver-gpu-ocp | helm | dra-driver-nvidia-gpu | 0.4.1 | 1 | +| nvidia-dra-driver-gpu | helm | dra-driver-nvidia-gpu | 0.5.0 | 1 | +| nvidia-dra-driver-gpu-ocp | helm | dra-driver-nvidia-gpu | 0.5.0 | 1 | | nvsentinel | helm | nvsentinel | v1.20.0 | 6 | | prometheus-adapter | helm | prometheus-community/prometheus-adapter | 5.3.0 | 1 | | prometheus-adapter-ocp | helm | prometheus-community/prometheus-adapter | 5.3.0 | 1 | @@ -149,20 +149,20 @@ _No images extracted._ ### gpu-operator - `docker.io/library/busybox:1.38.0@sha256:dc2d74b28e4cf8984fa52af1f39bc7c3d9c73760b41a74d629f5d11b1ab28616` -- `nvcr.io/nvidia/cloud-native/dcgm:4.5.2-1-ubuntu22.04` -- `nvcr.io/nvidia/cloud-native/gdrdrv:v2.5.2` -- `nvcr.io/nvidia/cloud-native/k8s-cc-manager:v0.4.0` -- `nvcr.io/nvidia/cloud-native/k8s-driver-manager:v0.11.0` -- `nvcr.io/nvidia/cloud-native/k8s-mig-manager:v0.14.2` -- `nvcr.io/nvidia/cloud-native/nvidia-fs:2.27.3` -- `nvcr.io/nvidia/cloud-native/nvidia-sandbox-device-plugin:v0.0.3` -- `nvcr.io/nvidia/cloud-native/vgpu-device-manager:v0.4.2` -- `nvcr.io/nvidia/driver:580.173.02` -- `nvcr.io/nvidia/gpu-operator:v26.3.3` -- `nvcr.io/nvidia/k8s-device-plugin:v0.19.3` -- `nvcr.io/nvidia/k8s/container-toolkit:v1.19.1` -- `nvcr.io/nvidia/k8s/dcgm-exporter:4.5.3-4.8.2-distroless` -- `nvcr.io/nvidia/kubevirt-gpu-device-plugin:v1.5.0` +- `nvcr.io/nvidia/cloud-native/dcgm:4.6.0-1-ubuntu24.04` +- `nvcr.io/nvidia/cloud-native/gdrdrv:v2.6` +- `nvcr.io/nvidia/cloud-native/k8s-cc-manager:v0.4.3` +- `nvcr.io/nvidia/cloud-native/k8s-driver-manager:v0.12.0` +- `nvcr.io/nvidia/cloud-native/k8s-mig-manager:v0.15.0` +- `nvcr.io/nvidia/cloud-native/nvidia-fs:2.29.4` +- `nvcr.io/nvidia/cloud-native/nvidia-sandbox-device-plugin:v0.0.5` +- `nvcr.io/nvidia/cloud-native/vgpu-device-manager:v0.5.0` +- `nvcr.io/nvidia/driver:595.91.07` +- `nvcr.io/nvidia/gpu-operator:v26.7.0` +- `nvcr.io/nvidia/k8s-device-plugin:v0.20.0` +- `nvcr.io/nvidia/k8s/container-toolkit:v1.20.0` +- `nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless` +- `nvcr.io/nvidia/kubevirt-gpu-device-plugin:v1.6.0` ### gpu-operator-ocp @@ -279,11 +279,11 @@ _No images extracted._ ### nvidia-dra-driver-gpu -- `registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.4.1` +- `registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.5.0` ### nvidia-dra-driver-gpu-ocp -- `registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.4.1` +- `registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.5.0` ### nvsentinel From 80390bca00e089ee54ffdfb4ec1ae1dae52fc4f0 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 09:38:03 -0700 Subject: [PATCH 03/16] test: regenerate catalog and stock-render golden digests for the version bump Signed-off-by: Yuan Chen --- .../testdata/catalog_parity_golden.yaml | 97 +++++++++---------- 1 file changed, 48 insertions(+), 49 deletions(-) diff --git a/pkg/recipe/testdata/catalog_parity_golden.yaml b/pkg/recipe/testdata/catalog_parity_golden.yaml index 1250f0aa0..06cffd0e0 100644 --- a/pkg/recipe/testdata/catalog_parity_golden.yaml +++ b/pkg/recipe/testdata/catalog_parity_golden.yaml @@ -3,52 +3,51 @@ # # One entry per leaf overlay: sha256 of its deterministically-marshalled # resolved recipe. A moved digest means that recipe's resolved bytes changed. -a100-aks-ubuntu-training-kubeflow: 47f2c604ad1491346fc4923dacafe6d636fce9fc62b7312e1972f2286e7a1184 -a100-any: 5abf30208c87a34a0f7a8ace6fb04b878de14233bb8691fe7344f1500985eed3 -a100-eks-ubuntu-training-kubeflow: 3cda91f7acec765b159ce307509479e30a9035acff615da06eab3a0b5d6d24c4 -a100-gke-cos-training-kubeflow: 7632830f5d0b5b669f6f4d509233c4314efc29f57152230d376fb2473eca7186 -a100-oke-ubuntu-training-kubeflow: 6619d20510945e05270ede31d1124313d45959f9cdfb189d63e5cd0f9eea7b16 -b200-any: c38ec9744d305fe7de0a5a590d445f028e38b0047786fe80c6a3c349ea95f02b -b200-gke-cos-inference-dynamo: a04ec4b0c5e24a12e08d29a7ed6f6cd8ff23cc53665fdca3f2ff89658507664d -b200-gke-cos-training-kubeflow: 62eec84f2f80c6f08fbfb2dadbd859e2ba07e787e43b58c5bdcc8360cf4aee71 -bcm-inference: 799569869ffff42841c5e758d6da1bc6f474af41a3cca5f74f806597e5be2f10 -gb200-any: 0dfa723aa4cc4850bea174f575e2853d4760c56035efb4acffa4d69b3ad2b886 -gb200-eks-ubuntu-inference-dynamo: c48c3c77fbc34e2f57223b97e5dc1fd2150fee3fae4a7ecbeae46394180f0c48 -gb200-eks-ubuntu-training-kubeflow: 36468d7c502b13e3ae6e441eecd0c6da7901fc2c89dfeac014d5df184cd74700 -gb200-eks-ubuntu-training-slurm: c660bbe8b1a97e250cdc7a61e75ea58be49dc96061626715f516bcbf885f2d13 -gb200-oke-ubuntu-inference-dynamo: 4ee02af889b1563108700d00575d418f0bd55f716404086579e16ece32700fe0 -gb200-oke-ubuntu-training-kubeflow: a2828e176878a36533d39a6ba720418bb6cb3eff34a6de7b2e7db83197248d69 -gb300-any: 4d22bb7186907f546733386a08efb32c8824552890b2e8d2c90cc717d201fb0d -gb300-eks-ubuntu-inference-dynamo: 5be01d4581d333a84a8597a77c192159a8bafe93104d6441ff8a0b6ad562c9c1 -gb300-eks-ubuntu-training-kubeflow: 33b8adf01bc622198ba9d46731c23efb88ab44991b84c8e37eea4ddf19aaeea9 -h100-aks-ubuntu-inference-dynamo: 46e0a581b1169cc1e5a4d3226aa46f4eb1a2fdad23ab37fba5c4825d773e80e5 -h100-aks-ubuntu-training-kubeflow: 3be5d233181d6dea5bf7127f554be827c579bac2fc893defca81621bbcc7ee77 -h100-aks-ubuntu-training-slurm: 2d797149b58ce853e7150d908be331c61dfdb477220fae9c15f6ebea22f9fe22 -h100-any: 2896754239e04cf8490a29c1a86b9a7d543cf982eb557c363f1e586668c1fe12 -h100-bcm-ubuntu-training: 85c125e8440059a5087e3835551edb0196bf9256f27f84cb4fa094b3517536e9 -h100-eks-ubuntu-inference-dynamo: 1a3f60c84c015754794edca31eb5abcef6444ed155e3470aca4bab4be3a7db3d -h100-eks-ubuntu-inference-nim: 53a96e375f75eed6508e57978e1e07fd05f740688dc4ca5a0e2fd04b90d716f3 -h100-eks-ubuntu-training-kubeflow: ecac42280c08d1eccaa49c31dee97009c803c360465d942de68a4d9786e8334c -h100-eks-ubuntu-training-slurm: 5fc03eb17bbb1934a78c03576d4aeeda8e723ca0937ca2fb0f6e0e40c19d38f2 -h100-gke-cos-inference-dynamo: 0c3d27435ece2300c0e88855f362d4e4cded31b9006c3d641560cab3fdffb2fd -h100-gke-cos-training-kubeflow: a6b36e216007c0be87bfadd22012ab0c25a94ce25a9a27e4d1c190bbaf63adb3 -h100-gke-cos-training-slurm: a4aaadffe6e17e6d01f2bdea68e5ea2d76fff2fc364b27ed811d9cf477763646 -h100-kind-inference-dynamo: 805b8224a75834e82d34cc8228e81a16e96a67b6b5742c9a329fc707c6deb6c8 -h100-kind-training-kubeflow: 5dd6d4efb2585c41c58dba1f373a5738c19be44618d109d5e31a53cf56482bd8 -h100-kind-training-slurm: 355c088589f5c948e025483d71468509addec60544b137653bf4df07f222ccc1 -h200-any: 15d07296d4050a66f0df218d3a41435541f69bb965c643869bc775e41988b4b5 -h200-eks-inference: 68b97d51d0cbc982564175114cb8626824e7643ac2fb6babf62073311c7eeb60 -h200-eks-training: 6db44e06178a0f74fab4d554406f988fe2e6c119b9523e458eb470ab3ed0cc8d -l40-any: abcbb63d3cd95efac239582f156b5af2c116079e3c48407fd275c889d0b2ad12 -l40s-any: 1bd1432c216e2e9f7b9ef061916e12e667aae1eb98fb3311857e2bc5c6dee44e -l40s-oke-inference: 0e0ef1669b6cb89ff3596a4b4c66f809b607d13ab264b1a02b82ad797c61b2bb -l40s-oke-training: bc8f2c4b08609739102969c4cc610045073834024b57b8121d2a715dbe8768c2 -monitoring-hpa: ca01840738fc06b47eb6713d630304ac2ca40693ed6bfbac9c57f7c8da4294a4 -ocp-inference-nim: fe0fc043ef8da714a0b3ff20b589faeba23a1bf7507380c3cf4c3f8ea6c37987 -ocp-training: 7c5beb2c2e001147b01e507a3d7e9cb1b2fa8c1cc20d6f86b02ebb271c00e390 -rtx-pro-6000-any: 945f6d4425ee52df4491fc74da78184a4d5f04f0204db4950694486993cc9244 -rtx-pro-6000-eks-ubuntu-inference-dynamo: 388abe59dc2fa07359455d387364e2e9fc3ed3922a9d0241604f67c8b65a1913 -rtx-pro-6000-eks-ubuntu-inference-nim: db4c9c06a32f435d04160a5e01198af6e6b42acbc90c7eb681378aa887cc3313 -rtx-pro-6000-eks-ubuntu-training-kubeflow: 5a1423754da34608921d125285e22e2ceaa053f494ae27b97892f1133e4a3f47 -rtx-pro-6000-lke-ubuntu-inference: c2a6382a95758cc36ebc77d8b157b2aecbdd9386849ac5153b94e0d657baa6dc -rtx-pro-6000-lke-ubuntu-training: 1fad1edc2ef174836e20cd4e4d4e64ff350d6ba4d57711e095318fee978cd398 +a100-aks-ubuntu-training-kubeflow: 52a19136ff27701be3ebd3291d0838b405dd4200b9ea2ad1f15f5e9231afc164 +a100-any: eedbde8040b6909da0e42d2af6345881ca5fcaacaa3ac83b222ba998c5c07c5e +a100-eks-ubuntu-training-kubeflow: 9918500dfc098857fa643a8427d378ad205fc7914566095c8678c2b837b93e7a +a100-gke-cos-training-kubeflow: e2dedc04e012cf16d146c953bc1f6a58237afd305e1978b58976d67d5695cb37 +a100-oke-ubuntu-training-kubeflow: c2f71c8a4bb59e2cca85fc007740eede1ded4a6514ec46071d7172f707c0fa13 +b200-any: 2d0d453bcdcd89d5db6b0aab435aad942458b108a7b0060a48e73c88a96e11ca +b200-gke-cos-inference-dynamo: 37147bb9daf7782bd58349c97ef01fa72b70fb46f1e01746d8725869a83ecfc6 +b200-gke-cos-training-kubeflow: e219956eaff20825678d485dd656e6071a6a91afc180af9abe54122d93191164 +bcm-inference: c608118b2f1c74e2d1d2d9defa90918f9da2cf6122afbf133d605f8a0154ea87 +gb200-any: 548063fe33335835470c1bbf9fb9ace134061d06a8910409e68d90423377a41d +gb200-eks-ubuntu-inference-dynamo: b15f4dd4cf5224ca76f7524f15611ed22910c4ee90334fe160594cb1592e9ecc +gb200-eks-ubuntu-training-kubeflow: fc6b234dd59ec815ba4bc67c695ff04057dbf3a233ba68662c54b96450ba035b +gb200-eks-ubuntu-training-slurm: 4b2007f13eb5c953620a3e18271e3f5b8d8a407258a36c1c47882012490d0470 +gb200-oke-ubuntu-inference-dynamo: 4ce2e85e13d125948e91614f0d07ebb70968ea0d93beec9feba03863f420f282 +gb200-oke-ubuntu-training-kubeflow: 3f0394a1b85b3d16147a31ecf55088d899c3b103ead7cb2f8a25183cb3a1575d +gb300-any: 10eb8b5cc815ca6a190412e8e344d261c0153cc65e2bf8266d89f6d821171da3 +gb300-eks-ubuntu-inference-dynamo: 9a4d41b7e6c57ec6cb44df5f6af64ca3f4d15a4568d6dfaaa16dfc0e5b791877 +gb300-eks-ubuntu-training-kubeflow: 460cb834803767f98f70f6bad7e006d22ebd79f83aa4c62f3feace0f896bd579 +h100-aks-ubuntu-inference-dynamo: e16910a1555ed867d5c85b63d6bea4861e585f1c4cbffeb81b9315fe04feb551 +h100-aks-ubuntu-training-kubeflow: 0ef4f87a06b668338e8416eb7b85e5ac63f472ebbd478c69db1a6e36d9cdac6a +h100-aks-ubuntu-training-slurm: f43f166e8d0af0447fa581e227fff3088d8a0d5efb9bac8cda4aff2f6cf84d04 +h100-any: 879ed75bd46ef020cc9f31eb03bceb49debe3719c2dcf567d072ac5b13ba3d2e +h100-bcm-ubuntu-training: 2d25d1fa54d969088215f4f95d2eabd2aead572a4223d2421f6d9ffaaf15be61 +h100-eks-ubuntu-inference-dynamo: 0b20a1afb8336d0fb7abe60dd5308cb2b76b96df2a4932fe18196468a08b1012 +h100-eks-ubuntu-inference-nim: 00460f237a30b03854875ce52a9e7a58217878192cfeb3f52745b26d1ee66c18 +h100-eks-ubuntu-training-kubeflow: 0940f0e8420302f91cd11f3d0451406de9259408245ef9410423ad09bb369766 +h100-eks-ubuntu-training-slurm: 939f6422a3b79e68b5a43225a838ecba9d2878d57cd9b4a98207d6d2860b1193 +h100-gke-cos-inference-dynamo: 904aeab7e68bce844fc9a72a9a79f27d28f77d044d510fdcbaecce596cefe983 +h100-gke-cos-training-kubeflow: 2ec3f85e22a551c48bca2a67af7211965c405f7a6fe7e4fa8be3a161311943a9 +h100-gke-cos-training-slurm: b74e224e7362f803da02ed60e090954d3c70d964cdc6db62df4bac4b5a060f70 +h100-kind-inference-dynamo: b17a7431bf9b5cd97211d343274324bdf7871b44b504f77bd575f54092c993db +h100-kind-training-kubeflow: 49f22bb0ba0b62c8c728cb801e99af9d4515ecbff4151100a0788bd3f4c1dcf2 +h100-kind-training-slurm: 891a37a0ff30b83bae3252eaae8305fb1fc4f29ddd3cafcb751f561308b738a7 +h200-any: 959608a2eb8060c6df2ef7caf453a1500f1adc8790b8c228851c603d15b6e908 +h200-eks-inference: dd2814a49106bfae9623bd310587ae4bcf378fba20039726796272fe23a571cc +h200-eks-training: cd7439667773a77b523a7337e672f4b8a9828bf88af618436094dc556746acb1 +l40s-any: 83b2b92d8b43e8f23754ff9608f02a6d64d7de28f8ccb3b7d6ae67679cffac93 +l40s-oke-inference: 9761d9172ba2c7561e0406392e2c0828450f6e3d5537c61bdaf5732d80fca487 +l40s-oke-training: 4b40b25e4a5620d3cea304df67209b397e3813190ee2ff4c7dd5798690ecd10d +monitoring-hpa: 7156deeffed71f32316b4a2501c771033d67dc2de3d3181311e985cc3625e557 +ocp-inference-nim: 08f9f5d03f030160eb6951bd238c24a400c41d62dcf721589abb3d265df8e3fc +ocp-training: 3998c76724c4189cc463e5cf473a3cd97d8c0eaf3299336be7f1ffac9cb11e6f +rtx-pro-6000-any: 62760c1f1e8287ae8c37b20d306546469bed84a742ea2b143aad52f942575f7e +rtx-pro-6000-eks-ubuntu-inference-dynamo: 2e14248538447738e7ea1699b0c0730dfd1e18b8aa8588ab0540903874e3bc2e +rtx-pro-6000-eks-ubuntu-inference-nim: b1c918b05e42118edcb51a164a9e251c944243d3e8d0127855ec361f8740ee52 +rtx-pro-6000-eks-ubuntu-training-kubeflow: cb833e1bcc917f105a106f4e36186ee55c665c641c22bd0a5da4206284b04daf +rtx-pro-6000-lke-ubuntu-inference: 1de10952036333b9f1d475b5ae212d54e4ed226ff3a44c17b13415bb708e5368 +rtx-pro-6000-lke-ubuntu-training: 296c4559de7977f238303e787407e57489a0dae8b24249f9f416a0425d4d48c7 From a7b8e3fca4aea97c05fe4a307dd2c755503d1e66 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 09:54:15 -0700 Subject: [PATCH 04/16] docs(recipes): record the GB200+EFA floor history on the driver pin Signed-off-by: Yuan Chen --- recipes/components/gpu-operator/values.yaml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/recipes/components/gpu-operator/values.yaml b/recipes/components/gpu-operator/values.yaml index de4866fd3..a5de768be 100644 --- a/recipes/components/gpu-operator/values.yaml +++ b/recipes/components/gpu-operator/values.yaml @@ -147,12 +147,17 @@ gfd: enabled: true driver: - # NVIDIA's recommended driver for the v26.7.0 chart; a single global - # pin covers H100/B200/GB200 EKS. Pinned rather than inherited from the - # chart default so a future chart bump is an explicit driver decision - # instead of a silent fleet-wide driver change, and so BOM/attestation - # inputs stay deterministic. Moves with the chart: holding an older - # driver against a newer chart is a combination upstream does not test. + # Single global pin covering H100/B200/GB200 EKS. Pinned rather than + # inherited from the chart default so a future chart bump is an explicit + # driver decision instead of a silent fleet-wide driver change, and so + # BOM/attestation inputs stay deterministic. + # + # History: the previous pin (580.173.02) was deliberately AHEAD of the + # v26.3.3 chart default (580.126.20) to clear a GB200+EFA floor. This + # value is the v26.7.0 chart default and is far above that floor, so the + # pin and the chart recommendation coincide again — but they are not the + # same thing. If a future chart default regresses below a GB200+EFA + # requirement, hold the pin ahead rather than following the chart. version: 595.91.07 enabled: true useOpenKernelModules: true From b81c9baaf84c0283a6fafe00dd1bc23427711027 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 10:16:09 -0700 Subject: [PATCH 05/16] fix(recipes): pin ComputeDomain numNodes and refresh stale version references gpu-operator v26.7.0 ships a stale copy of the ComputeDomain CRD in crds/ that marks numNodes required with no default. Helm installs crds/ only when absent and gpu-operator is ordered before nvidia-dra-driver-gpu, so on a fresh cluster the operator's copy wins and the slinky-slurm ComputeDomain CR is rejected with "spec.numNodes: Required value". Pin numNodes: 0 explicitly, which is valid under both CRD copies and is the correct value under IMEXDaemonsWithDNSNames. Also refresh version references the bump made stale: the GPU CI chart pin, a chainsaw comment mirroring the driver-validation rationale, the ccManager and toolkit-hardening comments, and a hand-written sample in the BOM page. Signed-off-by: Yuan Chen --- .settings.yaml | 2 +- docs/user/container-images.md | 2 +- .../manifests/nvidia-toolkit-hardening-aks.yaml | 2 +- recipes/components/gpu-operator/values.yaml | 5 +++-- .../slinky-slurm/manifests/compute-domain.yaml | 15 +++++++++++++++ .../cuj1-training/assert-bundle-scheduling.yaml | 2 +- 6 files changed, 22 insertions(+), 6 deletions(-) diff --git a/.settings.yaml b/.settings.yaml index 49161095c..86790489f 100644 --- a/.settings.yaml +++ b/.settings.yaml @@ -204,7 +204,7 @@ testing: # GPU CI runtime pins consumed through .github/actions/load-versions. # renovate: datasource=helm depName=gpu-operator depType=testing - gpu_operator_chart_version: 'v26.3.3' + gpu_operator_chart_version: 'v26.7.0' # Base image for the CI smoke-test snapshot-agent image. The agent binary is # static Go and detects GPUs driver-free via NFD/PCI, so it no longer needs a # CUDA base / nvidia-smi. Matches the NVIDIA static distroless base used for diff --git a/docs/user/container-images.md b/docs/user/container-images.md index dcf0735f1..245a5c983 100644 --- a/docs/user/container-images.md +++ b/docs/user/container-images.md @@ -448,7 +448,7 @@ Y = artifact attached, - = artifact absent, ? = could not probe. Image Sig SBOM Prov Notes -------------------------------------------------------------- --- ---- ---- ----- - registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.4.1 Y - - + registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.5.0 Y - - Summary: 1/1 signed · 0/1 SBOM · 0/1 provenance ``` diff --git a/recipes/components/gpu-operator/manifests/nvidia-toolkit-hardening-aks.yaml b/recipes/components/gpu-operator/manifests/nvidia-toolkit-hardening-aks.yaml index f8a440c85..885dbc00a 100644 --- a/recipes/components/gpu-operator/manifests/nvidia-toolkit-hardening-aks.yaml +++ b/recipes/components/gpu-operator/manifests/nvidia-toolkit-hardening-aks.yaml @@ -55,7 +55,7 @@ # device-request path open — the same posture as GPU-Operator-managed mode, # not a regression — so full multi-tenant isolation additionally requires # an admission policy restricting a /dev/null-backed mount whose destination -# is under /var/run/nvidia-container-devices (the pinned toolkit v1.19.1 +# is under /var/run/nvidia-container-devices (the pinned toolkit v1.20.0 # accepts the volume-mount device request only when the mount SOURCE is # /dev/null). secure-accelerator-access does not # exercise the volume-mounts path, so a green result is not proof of diff --git a/recipes/components/gpu-operator/values.yaml b/recipes/components/gpu-operator/values.yaml index a5de768be..d7cfd4c35 100644 --- a/recipes/components/gpu-operator/values.yaml +++ b/recipes/components/gpu-operator/values.yaml @@ -198,8 +198,9 @@ validator: nfd: enabled: false -# Confidential Compute Manager defaults to enabled in chart v26.3.x; keep -# it off until AICR has explicit CC-capable hardware support. +# Confidential Compute Manager defaults to enabled in the upstream chart +# (verified still true in v26.7.0); keep it off until AICR has explicit +# CC-capable hardware support. ccManager: enabled: false diff --git a/recipes/components/slinky-slurm/manifests/compute-domain.yaml b/recipes/components/slinky-slurm/manifests/compute-domain.yaml index 3bf024787..3af09f839 100644 --- a/recipes/components/slinky-slurm/manifests/compute-domain.yaml +++ b/recipes/components/slinky-slurm/manifests/compute-domain.yaml @@ -29,6 +29,21 @@ metadata: name: slinky-slurm-imex namespace: {{ .Release.Namespace }} spec: + # Pinned explicitly rather than relying on the CRD default. GPU Operator + # v26.7.0 ships its own copy of resource.nvidia.com_computedomains.yaml in + # crds/, and that copy is a stale snapshot: it lists numNodes as REQUIRED + # and carries no `default: 0`, where the DRA driver chart's copy makes it + # optional with a default. Helm installs crds/ only when the CRD is absent + # and never upgrades it, and gpu-operator is ordered before + # nvidia-dra-driver-gpu, so on a fresh cluster the operator's stricter copy + # wins and a CR omitting numNodes is rejected with "spec.numNodes: Required + # value". Setting it here is valid under both copies. + # + # 0 is the correct value, not merely a placeholder: with + # IMEXDaemonsWithDNSNames=true (the DRA driver default) each IMEX daemon + # starts immediately instead of waiting for a quorum. Matches + # buildComputeDomain() in validators/performance/nccl_all_reduce_bw_constraint.go. + numNodes: 0 channel: allocationMode: All resourceClaimTemplate: diff --git a/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml b/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml index 5b59ffdac..858d25528 100644 --- a/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml +++ b/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml @@ -39,7 +39,7 @@ daemonsets: # ── Driver: nvidia_peermem off on EKS (AWS EFA path uses aws-ofi-nccl) # nvidia_peermem only loads against Mellanox MOFED symbols; on AWS EFA -# (p4d/p5/p5e) it fails to load and v26.3.3's strict driver-validation +# (p4d/p5/p5e) it fails to load and the chart's strict driver-validation # init container blocks the rest of the GPU stack. NCCL multi-node on # EFA uses libfabric via aws-ofi-nccl, not nvidia_peermem. driver: From 84d235a3ca6f1d02aacb1040fa5557e3c8eac2ef Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 10:20:38 -0700 Subject: [PATCH 06/16] docs: align docs, demos and examples with the bumped component versions Verified each version-specific behavioral claim in Go comments against v26.7.0 / DRA 0.5.0 before updating the cited version: transformForDriverInstallDir still early-returns on "", clusterpolicy.yaml still accesses .Values.hostPaths.rootFS unconditionally, the ClusterPolicy CRD still types driverInstallDir as a string, _helpers.tpl still reads .Values.driver.manager.repository, operator.runtimeClass still defaults to nvidia, driverInstallDir still defaults to /run/nvidia/driver, and the DRA chart still defaults nvidiaDriverRoot to "/". The GKE Ubuntu driver-management note is de-versioned rather than re-pinned, since it is a support-matrix claim rather than a chart fact. Docs, demos and example recipes carried the old versions in copyable samples and in one exact-match constraint (== v26.3.3), which would have failed validation against a v26.7.0 deployment. Signed-off-by: Yuan Chen --- demos/dynamic.md | 6 +++--- demos/query.md | 4 ++-- docs/integrator/automation.md | 2 +- docs/integrator/data-flow.md | 2 +- docs/integrator/recipe-development.md | 2 +- docs/user/air-gap-mirror.md | 8 ++++---- docs/user/api-reference.md | 2 +- docs/user/cli-reference.md | 6 +++--- examples/recipes/aks-training.yaml | 4 ++-- ...ks-gb200-ubuntu-training-with-validation.yaml | 8 ++++---- examples/recipes/eks-training.yaml | 2 +- pkg/bundler/validations/checks.go | 16 ++++++++-------- pkg/client/v1/gpu_driver_state.go | 2 +- pkg/recipe/driver_root_lockstep_test.go | 4 ++-- 14 files changed, 34 insertions(+), 34 deletions(-) diff --git a/demos/dynamic.md b/demos/dynamic.md index 8c5caf331..a178fb171 100644 --- a/demos/dynamic.md +++ b/demos/dynamic.md @@ -98,7 +98,7 @@ cat ./bundle/010-gpu-operator/cluster-values.yaml # # Generated by AICR # --- # driver: -# version: 580.173.02 +# version: 595.91.07 ``` And `install.sh` applies it after the baked values, so it wins: @@ -121,7 +121,7 @@ grep -c 'version: 580' ./bundle/010-gpu-operator/values.yaml # b) present in cluster-values.yaml grep version ./bundle/010-gpu-operator/cluster-values.yaml -# version: 580.173.02 +# version: 595.91.07 # c) install.sh references cluster-values.yaml grep -q cluster-values.yaml ./bundle/010-gpu-operator/install.sh && echo wired @@ -193,7 +193,7 @@ rm -rf ./b-oci oras pull "$REF" -o ./b-oci (cd ./b-oci && aicr verify .) cat ./b-oci/010-gpu-operator/cluster-values.yaml -# version: 580.173.02 +# version: 595.91.07 ``` > ttl.sh is public; anything pushed is world-readable until it expires. Fine for a throwaway demo bundle, not for anything sensitive — push to your own registry by pointing `spec.bundle.output.target` (or `--output`) at it, e.g. `oci://registry.example.com/aicr-bundle:tag`. Transport is separate: `spec.bundle.registry.plainHTTP: true` only switches HTTP instead of HTTPS (enable it only when the registry is intentionally served over plain HTTP, such as a local test registry); leave it off for any HTTPS registry, and use `spec.bundle.registry.insecureTLS: true` to skip TLS verification against a self-signed HTTPS registry. diff --git a/demos/query.md b/demos/query.md index f98c23bcd..4ce7d559a 100644 --- a/demos/query.md +++ b/demos/query.md @@ -38,7 +38,7 @@ aicr query \ --selector components.gpu-operator.values.driver.version ``` -> `580.173.02` +> `595.91.07` Subtree — full driver block: @@ -54,7 +54,7 @@ maxParallelUpgrades: 5 rdma: enabled: false useOpenKernelModules: true -version: 580.173.02 +version: 595.91.07 ``` ## Differentiation: Same Selector, Different Criteria diff --git a/docs/integrator/automation.md b/docs/integrator/automation.md index a6d5e6585..1fd53d47f 100644 --- a/docs/integrator/automation.md +++ b/docs/integrator/automation.md @@ -231,7 +231,7 @@ spec: # Helm chart from upstream - repoURL: https://helm.ngc.nvidia.com/nvidia chart: gpu-operator - targetRevision: v26.3.3 + targetRevision: v26.7.0 helm: valueFiles: # Values live under the numbered bundle dir (NNN-/) diff --git a/docs/integrator/data-flow.md b/docs/integrator/data-flow.md index 833eb0453..b0e7007fc 100644 --- a/docs/integrator/data-flow.md +++ b/docs/integrator/data-flow.md @@ -634,7 +634,7 @@ spec: sources: # Helm chart from upstream - repoURL: https://helm.ngc.nvidia.com/nvidia - targetRevision: v26.3.3 + targetRevision: v26.7.0 chart: gpu-operator helm: valueFiles: diff --git a/docs/integrator/recipe-development.md b/docs/integrator/recipe-development.md index 4985364b6..a469decc3 100644 --- a/docs/integrator/recipe-development.md +++ b/docs/integrator/recipe-development.md @@ -888,7 +888,7 @@ spec: # recipes/registry.yaml - name: gpu-operator helm: - defaultVersion: v26.3.3 # Changed from v26.3.2 + defaultVersion: v26.7.0 # Changed from v26.3.3 ``` **Adding components:** diff --git a/docs/user/air-gap-mirror.md b/docs/user/air-gap-mirror.md index e495b804d..43f2544d3 100644 --- a/docs/user/air-gap-mirror.md +++ b/docs/user/air-gap-mirror.md @@ -112,7 +112,7 @@ metadata: name: aicr-images spec: images: - - name: nvcr.io/nvidia/gpu-operator:v26.3.3 + - name: nvcr.io/nvidia/gpu-operator:v26.7.0 - name: registry.k8s.io/nfd/node-feature-discovery:v0.19.0 # ... --- @@ -124,7 +124,7 @@ spec: charts: - name: gpu-operator repoURL: oci://ghcr.io/nvidia - version: v26.3.3 + version: v26.7.0 # ... ``` @@ -187,13 +187,13 @@ components: - name: aicr-images required: true images: - - nvcr.io/nvidia/gpu-operator:v26.3.3 + - nvcr.io/nvidia/gpu-operator:v26.7.0 - registry.k8s.io/nfd/node-feature-discovery:v0.19.0 # ... charts: - name: gpu-operator url: oci://ghcr.io/nvidia/gpu-operator - version: v26.3.3 + version: v26.7.0 namespace: gpu-operator # ... ``` diff --git a/docs/user/api-reference.md b/docs/user/api-reference.md index 2b875303a..c6d75c840 100644 --- a/docs/user/api-reference.md +++ b/docs/user/api-reference.md @@ -714,7 +714,7 @@ curl -X POST "http://localhost:8080/v1/bundle" \ "apiVersion": "aicr.run/v1alpha2", "kind": "RecipeResult", "componentRefs": [ - {"name": "gpu-operator", "type": "Helm", "chart": "gpu-operator", "source": "https://helm.ngc.nvidia.com/nvidia", "version": "v26.3.3", "namespace": "gpu-operator", "valuesFile": "components/gpu-operator/values.yaml"}, + {"name": "gpu-operator", "type": "Helm", "chart": "gpu-operator", "source": "https://helm.ngc.nvidia.com/nvidia", "version": "v26.7.0", "namespace": "gpu-operator", "valuesFile": "components/gpu-operator/values.yaml"}, {"name": "network-operator", "type": "Helm", "chart": "network-operator", "source": "https://helm.ngc.nvidia.com/nvidia", "version": "26.1.1", "namespace": "nvidia-network-operator", "valuesFile": "components/network-operator/values.yaml"} ], "deploymentOrder": ["gpu-operator", "network-operator"] diff --git a/docs/user/cli-reference.md b/docs/user/cli-reference.md index f957a7c04..a09316b46 100644 --- a/docs/user/cli-reference.md +++ b/docs/user/cli-reference.md @@ -2115,7 +2115,7 @@ The `--vendor-charts` flag pulls upstream Helm chart bytes into the bundle at bu my-bundle/ 001-gpu-operator/ Chart.yaml # wrapper, declares the vendored subchart - charts/gpu-operator-v26.3.3.tgz # vendored upstream tarball + charts/gpu-operator-v26.7.0.tgz # vendored upstream tarball values.yaml # values nested under the subchart name cluster-values.yaml # dynamic values, also nested install.sh # helm upgrade --install ./ ... @@ -2144,10 +2144,10 @@ kind: BundleProvenance vendoredCharts: - name: gpu-operator chart: gpu-operator - version: v26.3.3 + version: v26.7.0 repository: https://helm.ngc.nvidia.com/nvidia sha256: abc123... - tarballName: gpu-operator-v26.3.3.tgz + tarballName: gpu-operator-v26.7.0.tgz pullerVersion: helm-cli v3.20.2 ``` diff --git a/examples/recipes/aks-training.yaml b/examples/recipes/aks-training.yaml index 418788d30..cdd8d4dcc 100644 --- a/examples/recipes/aks-training.yaml +++ b/examples/recipes/aks-training.yaml @@ -52,7 +52,7 @@ componentRefs: chart: gpu-operator type: Helm source: https://helm.ngc.nvidia.com/nvidia - version: v26.3.3 + version: v26.7.0 valuesFile: components/gpu-operator/values-aks-training.yaml dependencyRefs: - nfd @@ -151,7 +151,7 @@ componentRefs: chart: dra-driver-nvidia-gpu type: Helm source: oci://registry.k8s.io/dra-driver-nvidia/charts - version: 0.4.1 + version: 0.5.0 valuesFile: components/nvidia-dra-driver-gpu/values.yaml overrides: controller: diff --git a/examples/recipes/eks-gb200-ubuntu-training-with-validation.yaml b/examples/recipes/eks-gb200-ubuntu-training-with-validation.yaml index c9b91a3f8..49bbe8429 100644 --- a/examples/recipes/eks-gb200-ubuntu-training-with-validation.yaml +++ b/examples/recipes/eks-gb200-ubuntu-training-with-validation.yaml @@ -61,9 +61,9 @@ validation: deployment: constraints: - name: gpu-operator.version - value: "== v26.3.3" + value: "== v26.7.0" severity: warning - remediation: "Update GPU Operator to v26.3.3" + remediation: "Update GPU Operator to v26.7.0" checks: - expected-resources @@ -100,7 +100,7 @@ componentRefs: chart: gpu-operator type: Helm source: https://helm.ngc.nvidia.com/nvidia - version: v26.3.3 + version: v26.7.0 valuesFile: components/gpu-operator/values-eks-training.yaml expectedResources: - kind: Deployment @@ -129,7 +129,7 @@ componentRefs: chart: dra-driver-nvidia-gpu type: Helm source: oci://registry.k8s.io/dra-driver-nvidia/charts - version: 0.4.1 + version: 0.5.0 valuesFile: components/nvidia-dra-driver-gpu/values.yaml dependencyRefs: - gpu-operator diff --git a/examples/recipes/eks-training.yaml b/examples/recipes/eks-training.yaml index e4dc850a9..8de030f01 100644 --- a/examples/recipes/eks-training.yaml +++ b/examples/recipes/eks-training.yaml @@ -41,7 +41,7 @@ componentRefs: chart: gpu-operator type: Helm source: https://helm.ngc.nvidia.com/nvidia - version: v26.3.3 + version: v26.7.0 valuesFile: components/gpu-operator/values-eks-training.yaml dependencyRefs: - cert-manager diff --git a/pkg/bundler/validations/checks.go b/pkg/bundler/validations/checks.go index 22765a67a..2a2afe6fd 100644 --- a/pkg/bundler/validations/checks.go +++ b/pkg/bundler/validations/checks.go @@ -464,7 +464,7 @@ func driverAbsentRemedy(service recipe.CriteriaServiceType, os recipe.CriteriaOS "DaemonSet alongside it; see " + "docs/integrator/gke-gpu-setup.md." case recipe.CriteriaOSUbuntu: - // The pinned GPU Operator (v26.3.3) supports driver management + // The pinned GPU Operator supports driver management // on GKE only on Ubuntu node images with containerd. return "On GKE Ubuntu node images the GPU Operator can manage " + "the driver: bundle in GPU-Operator-managed mode: " + @@ -643,11 +643,11 @@ func effectiveComponentValues(ctx context.Context, recipeResult *recipe.RecipeRe // path.Clean'd (trailing-slash spellings compare equal, mirroring // pkg/recipe/driver_root_lockstep_test.go), declared empty string → // the default (the operator's own transformForDriverInstallDir treats -// "" identically to the default, gpu-operator v26.3.3). An explicitly +// "" identically to the default, gpu-operator v26.7.0). An explicitly // null or non-map hostPaths section is rejected with a blocking // message: Helm null-coalescing deletes a null key together with its // chart defaults, so the chart's unconditional .Values.hostPaths.rootFS -// access (clusterpolicy.yaml, v26.3.3) fails at install. A declared +// access (clusterpolicy.yaml, v26.7.0) fails at install. A declared // value that cleans to a relative path is rejected too — host-path // mounts require absolute paths. func resolveInstallDir(values map[string]any, componentName string) (string, bool, []string) { @@ -680,7 +680,7 @@ func resolveInstallDir(values map[string]any, componentName string) (string, boo // rejected rather than silently defaulted: the emitted values // would carry it verbatim, and the pinned ClusterPolicy CRD // types hostPaths.driverInstallDir as a string (gpu-operator - // v26.3.3 nvidia.com_clusterpolicies.yaml), so the install + // v26.7.0 nvidia.com_clusterpolicies.yaml), so the install // fails while a defaulted check would have validated against // /run/nvidia/driver instead. return installDir, false, []string{fmt.Sprintf( @@ -693,7 +693,7 @@ func resolveInstallDir(values map[string]any, componentName string) (string, boo if dir == "" { // Intentionally default-equivalent: the operator's own // transformForDriverInstallDir early-returns on "" exactly like - // the default (gpu-operator v26.3.3, controllers/object_controls.go). + // the default (gpu-operator v26.7.0, controllers/object_controls.go). return installDir, false, nil } cleaned := path.Clean(dir) @@ -716,7 +716,7 @@ func resolveInstallDir(values map[string]any, componentName string) (string, boo // resolveDRARoot resolves the effective nvidia-dra-driver-gpu // nvidiaDriverRoot for Rule 2. Only a genuinely ABSENT key falls back to -// the chart-default assumption ("/", DRA chart v0.4.1 values.yaml). A +// the chart-default assumption ("/", DRA chart v0.5.0 values.yaml). A // present null, empty-string, or non-string value is rejected: unlike the // gpu-operator's driverInstallDir (where "" is default-equivalent, see // resolveInstallDir), the DRA chart pipes the raw value through @@ -866,7 +866,7 @@ func nvsentinelDynamicGuardViolations(bundlerConfig *config.Config, componentNam // null-coalescing deletes the key together with its chart defaults, // so .Values.
is nil at render time and the gpu-operator // templates fail on unconditional field access (e.g. -// .Values.driver.manager.repository in _helpers.tpl, v26.3.3) — +// .Values.driver.manager.repository in _helpers.tpl, v26.7.0) — // ownership cannot be verified and the install would fail anyway. // A non-boolean toggle is rejected because the chart renders the // value unquoted, so YAML re-typing at install time can flip it to a @@ -1705,7 +1705,7 @@ func CheckNVSentinelDriverLabelDetectable(ctx context.Context, componentName str } // defaultRuntimeClassName is the shared chart default: the gpu-operator -// chart ships operator.runtimeClass: nvidia (v26.3.3, verified against +// chart ships operator.runtimeClass: nvidia (v26.7.0, verified against // the pinned chart values), and nvsentinel's metadata-collector subchart // ships runtimeClassName: "nvidia" (v1.9.0, charts/metadata-collector/ // values.yaml:31). Either side left unset therefore resolves to this diff --git a/pkg/client/v1/gpu_driver_state.go b/pkg/client/v1/gpu_driver_state.go index dd4d2da7e..59801164d 100644 --- a/pkg/client/v1/gpu_driver_state.go +++ b/pkg/client/v1/gpu_driver_state.go @@ -109,7 +109,7 @@ func driverAbsentRemedy(service recipe.CriteriaServiceType, os recipe.CriteriaOS "DaemonSet and pins the driver version — nothing to deploy " + "by hand; see docs/integrator/gke-gpu-setup.md." case recipe.CriteriaOSUbuntu: - // The pinned GPU Operator (v26.3.3) supports driver management + // The pinned GPU Operator supports driver management // on GKE only on Ubuntu node images with containerd. return "On GKE Ubuntu node images the GPU Operator can manage " + "the driver: bundle in GPU-Operator-managed mode: " + diff --git a/pkg/recipe/driver_root_lockstep_test.go b/pkg/recipe/driver_root_lockstep_test.go index 1372e6141..bd38abf53 100644 --- a/pkg/recipe/driver_root_lockstep_test.go +++ b/pkg/recipe/driver_root_lockstep_test.go @@ -74,7 +74,7 @@ import ( // **Why "explicitly set" matters for the lockstep case.** An empty value // falls through to the upstream chart's bundled default, which the test // cannot read — and per-component defaults differ (GPU Operator chart -// 26.3.3 defaults driverInstallDir to /run/nvidia/driver, but DRA chart +// 26.7.0 defaults driverInstallDir to /run/nvidia/driver, but DRA chart // 25.12.0 defaults nvidiaDriverRoot to /). Relying on chart defaults is // itself drift waiting to happen on the next chart bump, so when the // lockstep applies the test treats "not explicitly set on both" as a @@ -211,7 +211,7 @@ func TestDriverRootLockstep(t *testing.T) { t.Errorf( "overlay %q: both nvidia-dra-driver-gpu.nvidiaDriverRoot and gpu-operator.hostPaths.driverInstallDir are unset.\n"+ " Both must be set explicitly to the same path. Chart defaults differ across components\n"+ - " (gpu-operator chart 26.3.3: /run/nvidia/driver; dra chart 25.12.0: /), so an unset value\n"+ + " (gpu-operator chart 26.7.0: /run/nvidia/driver; dra chart 0.5.0: /), so an unset value\n"+ " is drift waiting to happen on the next chart bump.\n"+ " See issue #1087.", name) From 069d087f0663b6e765b1c694457985f40abb8691 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 10:39:41 -0700 Subject: [PATCH 07/16] test(recipe): guard ComputeDomain manifests against the CRD-overlap hazard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gpu-operator v26.7.0 and the standalone DRA driver both ship a computedomains.resource.nvidia.com CRD, and the copies differ: the operator's marks numNodes required with no default. Helm installs crds/ only when absent and gpu-operator is ordered first, so a fresh cluster gets the stricter copy and rejects any ComputeDomain CR omitting numNodes. An upgraded cluster masks this because DRA 0.4.1 already installed the permissive copy, so no upgrade-path e2e would catch a regression here. The guard ignores comment lines: these manifests legitimately discuss "spec.numNodes: Required value" in prose, and a naive substring scan matched that instead of the key — passing even with the key deleted. Verified by deleting numNodes and confirming the test fails, then restoring it. Also fails closed when the walk matches no manifests, so it cannot go silently inert. Signed-off-by: Yuan Chen --- pkg/recipe/computedomain_numnodes_test.go | 124 ++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 pkg/recipe/computedomain_numnodes_test.go diff --git a/pkg/recipe/computedomain_numnodes_test.go b/pkg/recipe/computedomain_numnodes_test.go new file mode 100644 index 000000000..b2cbe4bda --- /dev/null +++ b/pkg/recipe/computedomain_numnodes_test.go @@ -0,0 +1,124 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package recipe + +import ( + "io/fs" + "regexp" + "strings" + "testing" +) + +// numNodesKeyRE matches an indented mapping key. Anchored per line so a key +// inside a comment or a quoted error string cannot satisfy it. +var numNodesKeyRE = regexp.MustCompile(`(?m)^[ \t]+numNodes[ \t]*:`) + +// hasYAMLKey reports whether content declares key as a real mapping key, +// ignoring comment lines. +func hasYAMLKey(content, key string) bool { + var b strings.Builder + for _, line := range strings.Split(content, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "#") { + continue + } + b.WriteString(line) + b.WriteString("\n") + } + if key != "numNodes" { + panic("hasYAMLKey: only numNodes is supported") + } + return numNodesKeyRE.MatchString(b.String()) +} + +// TestComputeDomainManifestsSetNumNodes guards the fresh-install CRD-overlap +// hazard introduced by GPU Operator v26.7.0. +// +// Two charts in the catalog ship a CRD named computedomains.resource.nvidia.com: +// the standalone nvidia-dra-driver-gpu chart, and — new in gpu-operator +// v26.7.0 — the GPU Operator chart. The two copies are NOT identical. The +// operator's is a stale snapshot that lists numNodes as required and supplies +// no `default: 0`; the DRA driver's makes it optional with a default. +// +// Helm installs crds/ only when the CRD is absent and never upgrades it, and +// gpu-operator is ordered before nvidia-dra-driver-gpu. So on a FRESH cluster +// the operator's stricter copy is the one that lands, and any ComputeDomain CR +// omitting spec.numNodes is rejected by the API server with +// "spec.numNodes: Required value". Structural defaulting cannot rescue it +// because that copy carries no default. Neither chart installs a webhook that +// could supply the field. +// +// An UPGRADED cluster masks this: it already has the permissive copy installed +// by DRA 0.4.1, so the CR still admits. That asymmetry is why this is a unit +// guard rather than something an upgrade-path e2e would catch. +// +// The invariant: every ComputeDomain CR shipped in the catalog must set +// spec.numNodes explicitly, so it is valid under BOTH CRD copies regardless of +// which chart installed the CRD first. +// +// See PR #2439 and issue #1087 for the driver-root analogue of this +// cross-component coupling problem. +func TestComputeDomainManifestsSetNumNodes(t *testing.T) { + t.Parallel() + + efs := GetEmbeddedFS() + + var checked int + err := fs.WalkDir(efs, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".yaml") { + return nil + } + raw, readErr := efs.ReadFile(path) + if readErr != nil { + return readErr + } + content := string(raw) + if !strings.Contains(content, "kind: ComputeDomain") { + return nil + } + checked++ + + // The manifests are Helm templates, so a full YAML parse is not + // available. Strip comment lines FIRST — the surrounding prose in + // these files legitimately discusses "spec.numNodes: Required value", + // and a naive substring scan matches that instead of the real key, + // producing a guard that passes even when the key is deleted. + if !hasYAMLKey(content, "numNodes") { + t.Errorf("%s declares kind: ComputeDomain but does not set spec.numNodes.\n"+ + " GPU Operator v26.7.0 ships a ComputeDomain CRD copy that marks numNodes\n"+ + " REQUIRED with no default, and it is installed before the DRA driver's\n"+ + " permissive copy. On a fresh cluster this CR is rejected at admission with\n"+ + " \"spec.numNodes: Required value\".\n"+ + " Set numNodes explicitly (0 is correct under IMEXDaemonsWithDNSNames=true,\n"+ + " the DRA driver default, where each IMEX daemon starts without waiting for\n"+ + " a quorum). See PR #2439.", path) + } + return nil + }) + if err != nil { + t.Fatalf("walking embedded recipes: %v", err) + } + + // Fail closed on a vacuous pass: if the walk matched nothing, the guard is + // silently inert and a regression would go unnoticed. + if checked == 0 { + t.Fatal("no ComputeDomain manifests found in the embedded recipes — " + + "this guard is vacuous. Either the manifests moved, or the embed " + + "pattern no longer covers them.") + } + t.Logf("verified %d ComputeDomain manifest(s) set spec.numNodes", checked) +} From 04dded08edd006acd1c27899e45ae344374918a9 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 10:59:55 -0700 Subject: [PATCH 08/16] fix: correct misspell lint failure and finish the version-reference refresh misspell rejects "analogue"; the US spelling is required by the lint gate. Three stale references the earlier refresh missed: - demos/dynamic.md grepped for 'version: 580' to prove the dynamic value had been removed from baked values. With the pin at 595.91.07 that grep returns 0 whether or not the substitution worked, so the check no longer proved anything. The chainsaw suite it mirrors uses a 999.99.99 sentinel and is unaffected. - the driver-root lockstep comment still named DRA chart 25.12.0 while its own error text had been updated to 0.5.0. - the AKS setup guide still named toolkit v1.19.1 while the manifest it documents now names v1.20.0. Signed-off-by: Yuan Chen --- demos/dynamic.md | 2 +- docs/integrator/aks-gpu-setup.md | 2 +- pkg/recipe/computedomain_numnodes_test.go | 2 +- pkg/recipe/driver_root_lockstep_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/demos/dynamic.md b/demos/dynamic.md index a178fb171..f2c1be1b1 100644 --- a/demos/dynamic.md +++ b/demos/dynamic.md @@ -116,7 +116,7 @@ Three quick checks, the same ones the chainsaw suite enforces: ```shell # a) gone from the baked values.yaml (no match) -grep -c 'version: 580' ./bundle/010-gpu-operator/values.yaml +grep -c 'version: 595' ./bundle/010-gpu-operator/values.yaml # 0 # b) present in cluster-values.yaml diff --git a/docs/integrator/aks-gpu-setup.md b/docs/integrator/aks-gpu-setup.md index 9f57fc9e1..83bf850b5 100644 --- a/docs/integrator/aks-gpu-setup.md +++ b/docs/integrator/aks-gpu-setup.md @@ -399,7 +399,7 @@ required so the device plugin's volume-mounts allocation strategy still works for legitimately allocated pods, but it leaves the **volume-mounts device-request path open** (a pod that declares a `/dev/null`-backed mount whose destination is under `/var/run/nvidia-container-devices` can still select -devices — the pinned toolkit v1.19.1 accepts the volume-mount device request +devices — the pinned toolkit v1.20.0 accepts the volume-mount device request only when the mount *source* is `/dev/null`). This is the same posture as GPU-Operator-managed mode — not a regression — but it means full multi-tenant isolation additionally requires an admission policy restricting a diff --git a/pkg/recipe/computedomain_numnodes_test.go b/pkg/recipe/computedomain_numnodes_test.go index b2cbe4bda..657051da6 100644 --- a/pkg/recipe/computedomain_numnodes_test.go +++ b/pkg/recipe/computedomain_numnodes_test.go @@ -67,7 +67,7 @@ func hasYAMLKey(content, key string) bool { // spec.numNodes explicitly, so it is valid under BOTH CRD copies regardless of // which chart installed the CRD first. // -// See PR #2439 and issue #1087 for the driver-root analogue of this +// See PR #2439 and issue #1087 for the driver-root analog of this // cross-component coupling problem. func TestComputeDomainManifestsSetNumNodes(t *testing.T) { t.Parallel() diff --git a/pkg/recipe/driver_root_lockstep_test.go b/pkg/recipe/driver_root_lockstep_test.go index bd38abf53..e1c83fdc7 100644 --- a/pkg/recipe/driver_root_lockstep_test.go +++ b/pkg/recipe/driver_root_lockstep_test.go @@ -75,7 +75,7 @@ import ( // falls through to the upstream chart's bundled default, which the test // cannot read — and per-component defaults differ (GPU Operator chart // 26.7.0 defaults driverInstallDir to /run/nvidia/driver, but DRA chart -// 25.12.0 defaults nvidiaDriverRoot to /). Relying on chart defaults is +// 0.5.0 defaults nvidiaDriverRoot to /). Relying on chart defaults is // itself drift waiting to happen on the next chart bump, so when the // lockstep applies the test treats "not explicitly set on both" as a // failure. From aee6c8f742f759954650eb7cd3ecec4981a6517c Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 11:36:39 -0700 Subject: [PATCH 09/16] test: scope the ComputeDomain guard per YAML document MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guard matched numNodes anywhere in the file. A multi-document manifest where one ComputeDomain sets the key and a second omits it would pass while still failing admission, as would an unrelated resource carrying a numNodes key. No such manifest exists today; scoping per document means adding one cannot silently bypass the guard. Two independent reviews raised this. Dropping the whole-file helper also removes its fake generic signature — it took a key parameter, ignored it in the regex, and panicked on anything but "numNodes". Controls verified both ways: deleting numNodes fails, and appending a second ComputeDomain document without it fails naming document 2. Restoring passes. Also updates two comments in checks_test.go that mirror comments in checks.go which this branch already moved to v26.7.0; the claims were verified against the v26.7.0 chart and still hold. Signed-off-by: Yuan Chen --- pkg/bundler/validations/checks_test.go | 4 +- pkg/recipe/computedomain_numnodes_test.go | 55 ++++++++++++++--------- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/pkg/bundler/validations/checks_test.go b/pkg/bundler/validations/checks_test.go index c72760c32..833f1718f 100644 --- a/pkg/bundler/validations/checks_test.go +++ b/pkg/bundler/validations/checks_test.go @@ -1330,7 +1330,7 @@ func TestCheckDriverOwnershipCoherence(t *testing.T) { // the key together with its chart defaults, so .Values.driver // is nil at render time and the chart's unconditional field // accesses (_helpers.tpl .Values.driver.manager.repository, - // v26.3.3) fail at install. Reject rather than default. The + // v26.7.0) fail at install. Reject rather than default. The // reachable vector is a top-level --set-json null: the typed // merge assigns it verbatim (mergeTypedValueByPath), while the // recipe-side overlay merge drops nil-valued keys before the @@ -1357,7 +1357,7 @@ func TestCheckDriverOwnershipCoherence(t *testing.T) { // hostPaths: null is the same hazard as a null driver/toolkit // section: Helm null-coalescing deletes the chart defaults and // clusterpolicy.yaml's unconditional .Values.hostPaths.rootFS - // access fails at install (v26.3.3). + // access fails at install (v26.7.0). name: "--set-json hostPaths=null → rejected", recipeResult: result("", aks, gpuOpRef(driverOn())), bundlerConfig: config.NewConfig(config.WithValueOverridesTypedPaths([]config.TypedComponentPath{ diff --git a/pkg/recipe/computedomain_numnodes_test.go b/pkg/recipe/computedomain_numnodes_test.go index 657051da6..b190b3a61 100644 --- a/pkg/recipe/computedomain_numnodes_test.go +++ b/pkg/recipe/computedomain_numnodes_test.go @@ -25,21 +25,41 @@ import ( // inside a comment or a quoted error string cannot satisfy it. var numNodesKeyRE = regexp.MustCompile(`(?m)^[ \t]+numNodes[ \t]*:`) -// hasYAMLKey reports whether content declares key as a real mapping key, -// ignoring comment lines. -func hasYAMLKey(content, key string) bool { - var b strings.Builder - for _, line := range strings.Split(content, "\n") { - if strings.HasPrefix(strings.TrimSpace(line), "#") { +// computeDomainDocsMissingNumNodes returns the 0-based indexes of YAML +// documents that declare kind: ComputeDomain without a numNodes key. +// +// Scoped per document rather than per file. A multi-document manifest where one +// ComputeDomain sets numNodes and a second omits it would satisfy a whole-file +// scan while still failing admission, and so would an unrelated resource that +// happens to carry a numNodes key. No such manifest exists in the catalog +// today; the guard is document-scoped so that adding one cannot silently +// bypass it. +// +// Comment lines are stripped first: these manifests legitimately discuss +// "spec.numNodes: Required value" in prose, and a naive substring scan matches +// that instead of the real key, passing even when the key is deleted. +// +// A full YAML parse is unavailable — the manifests are Helm templates and +// contain {{ }} expressions that no YAML parser accepts. +func computeDomainDocsMissingNumNodes(content string) []int { + var missing []int + for i, doc := range strings.Split(content, "\n---") { + if !strings.Contains(doc, "kind: ComputeDomain") { continue } - b.WriteString(line) - b.WriteString("\n") - } - if key != "numNodes" { - panic("hasYAMLKey: only numNodes is supported") + var b strings.Builder + for _, line := range strings.Split(doc, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "#") { + continue + } + b.WriteString(line) + b.WriteString("\n") + } + if !numNodesKeyRE.MatchString(b.String()) { + missing = append(missing, i) + } } - return numNodesKeyRE.MatchString(b.String()) + return missing } // TestComputeDomainManifestsSetNumNodes guards the fresh-install CRD-overlap @@ -92,20 +112,15 @@ func TestComputeDomainManifestsSetNumNodes(t *testing.T) { } checked++ - // The manifests are Helm templates, so a full YAML parse is not - // available. Strip comment lines FIRST — the surrounding prose in - // these files legitimately discusses "spec.numNodes: Required value", - // and a naive substring scan matches that instead of the real key, - // producing a guard that passes even when the key is deleted. - if !hasYAMLKey(content, "numNodes") { - t.Errorf("%s declares kind: ComputeDomain but does not set spec.numNodes.\n"+ + for _, idx := range computeDomainDocsMissingNumNodes(content) { + t.Errorf("%s: YAML document %d declares kind: ComputeDomain but does not set spec.numNodes.\n"+ " GPU Operator v26.7.0 ships a ComputeDomain CRD copy that marks numNodes\n"+ " REQUIRED with no default, and it is installed before the DRA driver's\n"+ " permissive copy. On a fresh cluster this CR is rejected at admission with\n"+ " \"spec.numNodes: Required value\".\n"+ " Set numNodes explicitly (0 is correct under IMEXDaemonsWithDNSNames=true,\n"+ " the DRA driver default, where each IMEX daemon starts without waiting for\n"+ - " a quorum). See PR #2439.", path) + " a quorum). See PR #2439.", path, idx) } return nil }) From 82803ca9fc3e4b6cec234a0174277188f51007ab Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 11:43:05 -0700 Subject: [PATCH 10/16] refactor(recipes): replace no-op useOpenKernelModules with kernelModuleType The chart deprecated useOpenKernelModules and made it a no-op; the identical NOTE appears in v26.3.3 and v26.7.0 values.yaml, so the key had already stopped having any effect before this pin moved. Our clusters have been running the chart default kernelModuleType: auto regardless of what that line said. Pinning "auto" records today's effective behaviour rather than changing it: it is the chart default on both versions and the fallback the OCP ClusterPolicy template already renders. Setting "open" would pin the intent the dead key expressed, but that is a real behaviour change wherever auto currently selects proprietary, so it needs its own validation rather than riding a version bump. Render goldens move for 46 of 48 leaves. The two that do not, ocp-inference-nim and ocp-training, consume gpu-operator-ocp and its separate values file. Catalog goldens are unchanged because values content affects rendering but not recipe resolution. Signed-off-by: Yuan Chen --- recipes/components/gpu-operator/values.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/recipes/components/gpu-operator/values.yaml b/recipes/components/gpu-operator/values.yaml index d7cfd4c35..e71d8a7e5 100644 --- a/recipes/components/gpu-operator/values.yaml +++ b/recipes/components/gpu-operator/values.yaml @@ -160,7 +160,23 @@ driver: # requirement, hold the pin ahead rather than following the chart. version: 595.91.07 enabled: true - useOpenKernelModules: true + # Replaces useOpenKernelModules, which the chart deprecated and made a + # no-op (identical NOTE in v26.3.3 and v26.7.0 values.yaml), so the old + # key had already stopped having any effect before this pin moved. + # + # "auto" pins today's effective behaviour, not a change: it is the chart + # default both before and after this bump, and it is what the OCP + # ClusterPolicy template already falls back to + # (components/gpu-operator-ocp/manifests/clusterpolicy.yaml). Per the + # ClusterPolicy CRD, auto selects the recommended module type from the + # host's GPU devices and driver branch — which yields open modules on + # Blackwell/Grace, where they are required. + # + # Setting "open" instead would pin the intent the dead key expressed, but + # that IS a behaviour change on any hardware where auto currently selects + # proprietary, so it needs its own validation rather than riding a version + # bump. + kernelModuleType: auto maxParallelUpgrades: 5 rdma: # Default off: nvidia_peermem only loads against Mellanox MOFED From 7472712d6eae89f97049b098260e79f8378d53c6 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 12:07:41 -0700 Subject: [PATCH 11/16] fix: make the ComputeDomain guard path-aware and update kernelModuleType consumers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guard matched numNodes anywhere in the ComputeDomain document, so moving the key to metadata.numNodes still passed while Kubernetes would reject the absent spec.numNodes. Matching a key without its parent is not a weaker check, it is the wrong check. The scanner now requires numNodes as a DIRECT CHILD of spec, walking indentation because the manifests are Helm templates no YAML parser accepts. Adds table cases covering every shape that has fooled a previous version of this scanner or must keep working: present, absent, under metadata, comment-only, nested under spec.channel, and templated. Plus a multi-document case asserting only the offending document is reported. Replacing useOpenKernelModules also broke four consumers that were not updated with it — a chainsaw bundle assertion (which fails CLI E2E), the OpenAPI response example, the query demo output, and the OCP values comment. Changing a values key requires sweeping its consumers, the same discipline applied to the version strings in this branch. Signed-off-by: Yuan Chen --- demos/query.md | 2 +- pkg/recipe/computedomain_numnodes_test.go | 157 +++++++++++++++--- .../components/gpu-operator-ocp/values.yaml | 2 +- .../assert-bundle-scheduling.yaml | 2 +- 4 files changed, 136 insertions(+), 27 deletions(-) diff --git a/demos/query.md b/demos/query.md index 4ce7d559a..3bda0bff9 100644 --- a/demos/query.md +++ b/demos/query.md @@ -53,7 +53,7 @@ enabled: true maxParallelUpgrades: 5 rdma: enabled: false -useOpenKernelModules: true +kernelModuleType: auto version: 595.91.07 ``` diff --git a/pkg/recipe/computedomain_numnodes_test.go b/pkg/recipe/computedomain_numnodes_test.go index b190b3a61..799bd92cc 100644 --- a/pkg/recipe/computedomain_numnodes_test.go +++ b/pkg/recipe/computedomain_numnodes_test.go @@ -21,41 +21,74 @@ import ( "testing" ) -// numNodesKeyRE matches an indented mapping key. Anchored per line so a key -// inside a comment or a quoted error string cannot satisfy it. -var numNodesKeyRE = regexp.MustCompile(`(?m)^[ \t]+numNodes[ \t]*:`) +// specKeyRE matches the document's top-level `spec:` mapping key. +var specKeyRE = regexp.MustCompile(`^(\s*)spec\s*:\s*$`) -// computeDomainDocsMissingNumNodes returns the 0-based indexes of YAML -// documents that declare kind: ComputeDomain without a numNodes key. +// numNodesChildRE matches `numNodes:` at a given exact indentation. +func numNodesChildRE(indent string) *regexp.Regexp { + return regexp.MustCompile(`^` + regexp.QuoteMeta(indent) + `numNodes\s*:`) +} + +// specHasNumNodes reports whether a single YAML document declares numNodes as a +// DIRECT CHILD of spec. // -// Scoped per document rather than per file. A multi-document manifest where one -// ComputeDomain sets numNodes and a second omits it would satisfy a whole-file -// scan while still failing admission, and so would an unrelated resource that -// happens to carry a numNodes key. No such manifest exists in the catalog -// today; the guard is document-scoped so that adding one cannot silently -// bypass it. +// Path-aware on purpose. An earlier version matched `numNodes:` anywhere in the +// document, which accepted `metadata.numNodes` — a key Kubernetes ignores, while +// the required `spec.numNodes` stays absent and admission still fails. Matching +// the key without its parent is not a weaker check, it is the wrong check. // // Comment lines are stripped first: these manifests legitimately discuss -// "spec.numNodes: Required value" in prose, and a naive substring scan matches -// that instead of the real key, passing even when the key is deleted. +// "spec.numNodes: Required value" in prose, and a scan that does not strip them +// matches that instead of the real key, passing even when the key is deleted. +// +// A full YAML parse is unavailable — the manifests are Helm templates containing +// {{ }} expressions that no YAML parser accepts — so this walks indentation. +func specHasNumNodes(doc string) bool { + var lines []string + for _, line := range strings.Split(doc, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "#") || strings.TrimSpace(line) == "" { + continue + } + lines = append(lines, line) + } + for i, line := range lines { + m := specKeyRE.FindStringSubmatch(line) + if m == nil { + continue + } + specIndent := m[1] + var childRE *regexp.Regexp + for _, sub := range lines[i+1:] { + subIndent := sub[:len(sub)-len(strings.TrimLeft(sub, " \t"))] + // Dedent to spec's level or shallower ends the spec mapping. + if len(subIndent) <= len(specIndent) { + break + } + if childRE == nil { + childRE = numNodesChildRE(subIndent) + } + if childRE.MatchString(sub) { + return true + } + } + } + return false +} + +// computeDomainDocsMissingNumNodes returns the 0-based indexes of YAML +// documents that declare kind: ComputeDomain without spec.numNodes. // -// A full YAML parse is unavailable — the manifests are Helm templates and -// contain {{ }} expressions that no YAML parser accepts. +// Scoped per document: a multi-document manifest where one ComputeDomain sets +// the key and a second omits it would satisfy a whole-file scan while still +// failing admission. No such manifest exists in the catalog today; the guard is +// document-scoped so adding one cannot silently bypass it. func computeDomainDocsMissingNumNodes(content string) []int { var missing []int for i, doc := range strings.Split(content, "\n---") { if !strings.Contains(doc, "kind: ComputeDomain") { continue } - var b strings.Builder - for _, line := range strings.Split(doc, "\n") { - if strings.HasPrefix(strings.TrimSpace(line), "#") { - continue - } - b.WriteString(line) - b.WriteString("\n") - } - if !numNodesKeyRE.MatchString(b.String()) { + if !specHasNumNodes(doc) { missing = append(missing, i) } } @@ -137,3 +170,79 @@ func TestComputeDomainManifestsSetNumNodes(t *testing.T) { } t.Logf("verified %d ComputeDomain manifest(s) set spec.numNodes", checked) } + +// TestComputeDomainScannerCases pins the scanner's behavior directly, so the +// catalog guard above cannot quietly stop discriminating if the catalog changes. +// Each case is a shape that has either fooled a previous version of this +// scanner or must keep working. +func TestComputeDomainScannerCases(t *testing.T) { + t.Parallel() + + const header = "apiVersion: resource.nvidia.com/v1beta1\nkind: ComputeDomain\n" + + tests := []struct { + name string + doc string + wantMissing bool + }{ + { + name: "spec.numNodes present", + doc: header + "metadata:\n name: cd\nspec:\n numNodes: 0\n channel:\n allocationMode: All\n", + }, + { + name: "spec.numNodes absent", + doc: header + "metadata:\n name: cd\nspec:\n channel:\n allocationMode: All\n", + wantMissing: true, + }, + { + // Regression: an earlier scanner matched numNodes anywhere in the + // document, so this passed while admission would still fail. + name: "numNodes under metadata, not spec", + doc: header + "metadata:\n name: cd\n numNodes: 0\nspec:\n channel:\n allocationMode: All\n", + wantMissing: true, + }, + { + // Regression: an earlier scanner did not strip comments, so the + // prose in the real manifest satisfied it even with the key gone. + name: "numNodes only mentioned in a comment", + doc: header + "metadata:\n name: cd\nspec:\n # numNodes: Required value\n channel:\n allocationMode: All\n", + wantMissing: true, + }, + { + name: "nested numNodes does not satisfy the direct-child rule", + doc: header + "metadata:\n name: cd\nspec:\n channel:\n numNodes: 0\n", + // numNodes exists but under spec.channel, not spec. + wantMissing: true, + }, + { + name: "templated value is acceptable", + doc: header + "metadata:\n name: cd\nspec:\n numNodes: {{ .Values.numNodes }}\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := !specHasNumNodes(tt.doc) + if got != tt.wantMissing { + t.Errorf("specHasNumNodes reported missing=%v, want %v\ndoc:\n%s", + got, tt.wantMissing, tt.doc) + } + }) + } +} + +// TestComputeDomainMultiDocument covers the per-document scoping: a file where +// one ComputeDomain is valid and a second is not must report only the second. +func TestComputeDomainMultiDocument(t *testing.T) { + t.Parallel() + + content := "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: unrelated\n" + + "\n---\napiVersion: resource.nvidia.com/v1beta1\nkind: ComputeDomain\nmetadata:\n name: ok\nspec:\n numNodes: 0\n" + + "\n---\napiVersion: resource.nvidia.com/v1beta1\nkind: ComputeDomain\nmetadata:\n name: bad\nspec:\n channel:\n allocationMode: All\n" + + missing := computeDomainDocsMissingNumNodes(content) + if len(missing) != 1 || missing[0] != 2 { + t.Errorf("missing documents = %v, want [2] (only the third document lacks spec.numNodes)", missing) + } +} diff --git a/recipes/components/gpu-operator-ocp/values.yaml b/recipes/components/gpu-operator-ocp/values.yaml index 1f090147d..8b931edbc 100644 --- a/recipes/components/gpu-operator-ocp/values.yaml +++ b/recipes/components/gpu-operator-ocp/values.yaml @@ -160,7 +160,7 @@ daemonsets: # operator.upgradeCRD: Helm chart setting, not a ClusterPolicy field. # operator.resources: Helm chart setting for the operator Deployment, not CR. # driver.version: OCP operator manages driver version via the certified driver container. -# driver.useOpenKernelModules: OCP uses pre-built driver containers from the certified catalog. +# driver.kernelModuleType: OCP uses pre-built driver containers from the certified catalog. # driver.maxParallelUpgrades: Use daemonsets.rollingUpdate.maxUnavailable instead. # devicePlugin.env: OCP sets device plugin env via the operator's own defaults. # validator: OCP operator manages validation internally. diff --git a/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml b/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml index 858d25528..c38d7c7d3 100644 --- a/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml +++ b/tests/chainsaw/cli/cuj1-training/assert-bundle-scheduling.yaml @@ -46,7 +46,7 @@ driver: enabled: true rdma: enabled: false - useOpenKernelModules: true + kernelModuleType: auto # ── GDRCopy: GPU-direct memory for high-performance training ───────── gdrcopy: From 44ee51cef92dba12ca6c67b4d96853cfadc7079d Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 15:53:59 -0700 Subject: [PATCH 12/16] fix(recipes): hold the GPU driver at 580.173.02 for P6e EFA compatibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NVIDIA documents R595 as backward incompatible with P6e-GB200 EFA. R595 drops NVreg_GrdmaPciTopoCheckOverride, the R580-era escape hatch, and substitutes a topology check requiring EFA in an IOMMU group or GPU and EFA under a shared root port. p6e-gb200/gb300 satisfies neither. Proven on p6e-gb300r.36xlarge, not inferred: running the NCCL NET workload under 595 with the preflight bypassed, both nodes selected aws-ofi-nccl/EFA/RDMA and then aborted, with the kernel logging "NVRM: dma-buf attach failed: topology not supported for mapping type FORCE_PCIE". The same two nodes passed NCCL NET at 43.16 GB/s under 580.173.02. Scope is the AWS P6e Grace-Blackwell EFA GPUDirect path. NVLS passed under 595 at 840 GB/s, as did deployment and conformance — NVLink never touches EFA. Holding here is a supported combination rather than an untested one: GPU Operator v26.7.0 lists 580.173.02 as supported (595.91.07 is only its default), and DRA driver 0.5.0 requires >= 580. The chart and DRA bumps are retained. This reverses an argument made earlier on this branch, that driver and chart should move together because holding one back invents an untested pairing. That reasoning did not account for the new driver being unsupported on hardware we ship. Signed-off-by: Yuan Chen --- docs/user/container-images.md | 2 +- recipes/components/gpu-operator/values.yaml | 41 +++++++++++++++------ 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/docs/user/container-images.md b/docs/user/container-images.md index 245a5c983..1c476cadb 100644 --- a/docs/user/container-images.md +++ b/docs/user/container-images.md @@ -157,7 +157,7 @@ _No images extracted._ - `nvcr.io/nvidia/cloud-native/nvidia-fs:2.29.4` - `nvcr.io/nvidia/cloud-native/nvidia-sandbox-device-plugin:v0.0.5` - `nvcr.io/nvidia/cloud-native/vgpu-device-manager:v0.5.0` -- `nvcr.io/nvidia/driver:595.91.07` +- `nvcr.io/nvidia/driver:580.173.02` - `nvcr.io/nvidia/gpu-operator:v26.7.0` - `nvcr.io/nvidia/k8s-device-plugin:v0.20.0` - `nvcr.io/nvidia/k8s/container-toolkit:v1.20.0` diff --git a/recipes/components/gpu-operator/values.yaml b/recipes/components/gpu-operator/values.yaml index e71d8a7e5..569d102d2 100644 --- a/recipes/components/gpu-operator/values.yaml +++ b/recipes/components/gpu-operator/values.yaml @@ -147,18 +147,37 @@ gfd: enabled: true driver: - # Single global pin covering H100/B200/GB200 EKS. Pinned rather than - # inherited from the chart default so a future chart bump is an explicit - # driver decision instead of a silent fleet-wide driver change, and so - # BOM/attestation inputs stay deterministic. + # HELD at 580.173.02 — deliberately BEHIND the v26.7.0 chart default of + # 595.91.07. Do not advance without EFA validation on P6e hardware. # - # History: the previous pin (580.173.02) was deliberately AHEAD of the - # v26.3.3 chart default (580.126.20) to clear a GB200+EFA floor. This - # value is the v26.7.0 chart default and is far above that floor, so the - # pin and the chart recommendation coincide again — but they are not the - # same thing. If a future chart default regresses below a GB200+EFA - # requirement, hold the pin ahead rather than following the chart. - version: 595.91.07 + # NVIDIA documents R595 as backward incompatible with P6e-GB200 EFA. R595 + # drops NVreg_GrdmaPciTopoCheckOverride (the R580-era escape hatch) and + # substitutes a topology check: EFA must sit in an IOMMU group, or GPU and + # EFA must share a root port. p6e-gb200/gb300 satisfies neither — measured + # on p6e-gb300r.36xlarge, where EFA (0000:97:00.0) has no iommu_group and + # sits under pci0000:84 while the GPU (0000:9d:00.0) is in group 0 under + # pci0000:98. + # + # The failure is proven, not inferred. Running the NCCL NET workload under + # 595 with the preflight bypassed, both nodes selected aws-ofi-nccl/EFA/RDMA + # and then aborted, with the kernel logging: + # NVRM: dma-buf attach failed: topology not supported for mapping type FORCE_PCIE + # Under 580.173.02 on the same two nodes, NCCL NET passed at 43.16 GB/s. + # + # Scope: the AWS P6e Grace-Blackwell + EFA GPUDirect path specifically. NVLS + # passed under 595 (840 GB/s), as did deployment and conformance; NVLink + # never touches EFA. + # + # Holding here is a supported combination, not an untested one: GPU Operator + # v26.7.0 lists 580.173.02 as supported (595.91.07 is merely its default), + # and DRA driver 0.5.0 requires >= 580. Pinned rather than inherited so a + # chart bump is an explicit driver decision, and so BOM/attestation inputs + # stay deterministic. + # + # Do not substitute a newer R580 patch without re-validating EFA — + # 580.173.02 is the exact measured baseline. Per-family overrides, so + # non-P6e families can advance independently, are tracked separately. + version: 580.173.02 enabled: true # Replaces useOpenKernelModules, which the chart deprecated and made a # no-op (identical NOTE in v26.3.3 and v26.7.0 values.yaml), so the old From 4cb104188a062a80b7ebad628a8d1d896d1f171e Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Sun, 30 Aug 2026 14:24:35 -0700 Subject: [PATCH 13/16] test: regenerate goldens after rebase onto main The rebase conflicted on both golden files; regenerating reapplies this branch's digests on top of main's current baseline. Signed-off-by: Yuan Chen --- pkg/bundler/testdata/stock_render_golden.yaml | 97 +++++++++---------- .../testdata/catalog_parity_golden.yaml | 12 +-- 2 files changed, 54 insertions(+), 55 deletions(-) diff --git a/pkg/bundler/testdata/stock_render_golden.yaml b/pkg/bundler/testdata/stock_render_golden.yaml index 61a272c15..105900fac 100644 --- a/pkg/bundler/testdata/stock_render_golden.yaml +++ b/pkg/bundler/testdata/stock_render_golden.yaml @@ -3,52 +3,51 @@ # # One entry per leaf overlay: a digest over its fully rendered helm-deployer # bundle tree (sorted relative paths paired with per-file content hashes). -a100-aks-ubuntu-training-kubeflow: b8161ed651945695164e296dbda3465d288ea19dcdf8d0ef6e84a1cc92af5cdd -a100-any: 8f6e700544dfd93a8be31a148badb32829924807aa1d13c2db2b80527e94c365 -a100-eks-ubuntu-training-kubeflow: 9d87274b22b1e3a3c35cee9f884ee00aa8bf2d98c75c7961be6d055ca8b2b4c4 -a100-gke-cos-training-kubeflow: 5bfae64e203fdc096e7e5df3d912d45fbf98e79543248c380c73e59562e059f6 -a100-oke-ubuntu-training-kubeflow: 5c46cfdaf69b6f38d2b3b377fc5641b5aa6c84b1030fe97426dc4874ed0a5819 -b200-any: b0a2e105824bd5042396bbdc91529a5986a62d7e0e2be5fd41a4ee31bc3ab24d -b200-gke-cos-inference-dynamo: 788a4749c063fab956cb133271aaec135148171fc761a34faf22e207d0356924 -b200-gke-cos-training-kubeflow: 0d72edc6fe724e87aaa278a60e31fd852b995b978ed7c7768b1ea1ed8cb71864 -bcm-inference: 0de8f37c9026519148af9d1f915e0cb35cef0799d39eb19a9f10736f54170742 -gb200-any: 5523fb162d04b85010ea14adcacde848823051530ecb2e0784d27bc5c0e10e1a -gb200-eks-ubuntu-inference-dynamo: f306a1fdd7d5fa81cdac39486928c0ebebf46690318d3279bd8a1258f4132698 -gb200-eks-ubuntu-training-kubeflow: f9de625240b1836d2d940cef2da8b989bd42d78e73273230ea7bd1b693345f9f -gb200-eks-ubuntu-training-slurm: 52c6c3858c36742d53f23ee36c5cb47859ef5b61ff2c9bbc49e932570b617f62 -gb200-oke-ubuntu-inference-dynamo: 4a8fa3f14fd6f8023311baccc03d12405d75d52b96159959a771ebca699e2cf6 -gb200-oke-ubuntu-training-kubeflow: 090e678f7ff902fdce9a06afcb5b30aea11a4b90c86979a6ca9059fbd9c14605 -gb300-any: 357198d4ae6a0e73528cf0ceda23258213e7eddba5f83aa25f133562bdaf65b0 -gb300-eks-ubuntu-inference-dynamo: f20aca19f018e8bbb9970276a6413c6cc898881ceb647c33c0ab7b8a9b078f3e -gb300-eks-ubuntu-training-kubeflow: 0d6b69f81dd4919d02ebad13324b5a1efe3b3efdb783bf91f5454eca194c57fc -h100-aks-ubuntu-inference-dynamo: 5b53e936929ee7ed8f07256472a2ec6885b7f0c752d003ce22a98b258fded50b -h100-aks-ubuntu-training-kubeflow: 5f5d9d468a54571bded9030a968a684a43779161dd0a487521283355bd19560d -h100-aks-ubuntu-training-slurm: 8544a761f71c8577952126a19c9a973b0890826d1680c89ec2ff63540d1b1ef6 -h100-any: 9a1e512c6d7fec5f9920ffc2d4b1f036fe59ba80322d4de286d931685bac70d5 -h100-bcm-ubuntu-training: cba1fe3491fe878683a4a99fe0af5bf08673b3d4ad21669351d6ce719b313ca3 -h100-eks-ubuntu-inference-dynamo: c42f8d3e2bad0aecaa952cf84a41e3603148c2ed2f7163c79c67397fca1ed187 -h100-eks-ubuntu-inference-nim: 6b6446a298885359b76f697970c33b80ace95de9cc0a4ab549fe58b8e579ec80 -h100-eks-ubuntu-training-kubeflow: e78076ebfd48bec60d410a3f0aa3da2d1cdff0ebec3f506b8b87e07110fd92f4 -h100-eks-ubuntu-training-slurm: f42961b910bec0b36b2b6ffcd5e2df6ec54f11755b94190b901884dc7cbff996 -h100-gke-cos-inference-dynamo: 1cd1373905cde68c6781925e354459c66e4094c8e6ede458bb75a377628fb5cb -h100-gke-cos-training-kubeflow: 6f0844b970b3706e3b5eb70dc3ae79c17119cf66c3127a7368813d345467bb83 -h100-gke-cos-training-slurm: edbbfa71d6dd096bf32d73412929f5779eafd5e172c990839098e02663a7b88d -h100-kind-inference-dynamo: b1734f6c44bff5e21da1d67b9f3a1b3d2de4fd6b48db411a19208c8bf7fda62d -h100-kind-training-kubeflow: 1c589e48629c86e5c8b7db3de708fb502bed7a4e48468a635e5da92cb83c3c1d -h100-kind-training-slurm: c8f780ca26d66c5191ba97409fdf9f49c5226eb78fee6ed54e06d05490b2215a -h200-any: 8f30849eac9a8489ce02ea195d538f6f76ce8afb0bcf1c8515843869bd304024 -h200-eks-inference: dbbeb66fc64382271916197fb24d951d10a5d4972c59aa7931438e5e3ec151f0 -h200-eks-training: 14db15267d453edbe3e4121fd1933cae633f2377c3aa6c198e97dc076f8cd464 -l40-any: da7bb3a2831930ddb224927d3ebfb5a5dd65d3c9a128c632aad413fb29c0e28b -l40s-any: 8c2f0d3ff1e2f0b5df3869650d3e74ec3808484f52949726bfd5db9c409c8199 -l40s-oke-inference: 69696a6ec37fc930d5fe728d28f979badb9ef9f6a4fa45a39a58b99c981c93ec -l40s-oke-training: 4afd8779e7c706530cd2936fdf0ce7537f86c9cbc019e02c4e05a6787cb31310 -monitoring-hpa: 1e4f15813317d9b472f66cedd8a280a1555a0a7e3142ba9c3dbfb014e578fbcd -ocp-inference-nim: 67cb04b6ad946fe56ce423972ec0692ebe8ad66f571f29e94536ae0124eee91c -ocp-training: 1082d9d2f5b8dd302d498ab9cb6c936dd6f95996f65e5ce41746ff84aefed04e -rtx-pro-6000-any: ca387e660b770165b828770102eaa0396b8a51f7157d78a2c714c631d5ecf70d -rtx-pro-6000-eks-ubuntu-inference-dynamo: 285ad9f64afc85fb0cf58269e18531a2457299453e35f1c0d9608f3a4f08288d -rtx-pro-6000-eks-ubuntu-inference-nim: ef70dad9934ed5c50478dcd772df08f0f0a41cc6c9a433edcdad5ce84fe92cf3 -rtx-pro-6000-eks-ubuntu-training-kubeflow: 1861f8389802b6592e667f217c45d965657c4f06875f98c53df6c906004280da -rtx-pro-6000-lke-ubuntu-inference: a876264218915286ebc002aa1134c5b64a25e9054f1fc88f59c9b68bcb80f2e0 -rtx-pro-6000-lke-ubuntu-training: a531171ba7c38877f1ade05c7cac7796d0907566ce7c6a3442502d51225d77b3 +a100-aks-ubuntu-training-kubeflow: 611e7dd4b638025c5ccc14bbec5afc24eda359b52f6ab82b0db0ce0235c80fa9 +a100-any: f3d55941c7fe2549a47ec14459773d26372488c6f8e4747fdf2682865535502c +a100-eks-ubuntu-training-kubeflow: f9a3471fcfb2a61e73f72767308a878a9deaba964603bda7a67445c033a93428 +a100-gke-cos-training-kubeflow: 4174ba7a47f024ff4f20918aea6d2ab237dc9e4c83e6bdf810259d0ca18d6f94 +a100-oke-ubuntu-training-kubeflow: ec7553d5d5c9884b75d626afbe2cfdc11891f417a5b8aabc49615eba66c1e0cb +b200-any: e4f8e29a45fdde19127181030eac2b0e97eb28c2e9d14fd28e30f62e31a6da54 +b200-gke-cos-inference-dynamo: 6cd7fd31f443982be752cc433e4143eeecd55a24fd0975ae1100eebc0df2c7fa +b200-gke-cos-training-kubeflow: fca1a9a7699f0811b3833b042e6068be759f366b29c1f4efcb2a63267cb04d9c +bcm-inference: aeadd0f0e6733f84cdacaad3f5d0610b45463aa049370a40b4543dcd6f7ece30 +gb200-any: 8355e5f6279cf96463c58ab5ced5237020742a9580fadff714da282489201690 +gb200-eks-ubuntu-inference-dynamo: c5ae52dbcc96704c07f5210c6fb604605651d08244a99e221b03232723f760e0 +gb200-eks-ubuntu-training-kubeflow: 67a7077f82ac9453c6c554e19cdba108154ad12bf94e41aaa7b81aceda1fcdb7 +gb200-eks-ubuntu-training-slurm: 28ad1fa1284dcd80f2547b0ad7f60d459cf47bfc9257bdfd9183dae50f9c79ab +gb200-oke-ubuntu-inference-dynamo: 4c816f06e392f17e6739cfaca9f72ad1200f6844eff593b98db7041ed02e55fe +gb200-oke-ubuntu-training-kubeflow: 37a4b534cd064726ce90841f2a5c1dd60521b930f50375d843a01370aa1476ab +gb300-any: 10258903d47723de166268d12c45e7f7587b3604ee9c1c63f853626615b0f18f +gb300-eks-ubuntu-inference-dynamo: cab5f3ba54cfb754534c8e7cbebe772d7fdd8a382bc4400615598c8abc4625eb +gb300-eks-ubuntu-training-kubeflow: 76ac339521a7a11ead88ee8c22e59cceb438990ea408c47ccafa2c3c61480506 +h100-aks-ubuntu-inference-dynamo: 14ea86cb775a62a4010ca3a003fd2cdbf2bbdc50184db5418d1458e126da5f34 +h100-aks-ubuntu-training-kubeflow: eafd5a02b9b87f5c543a60a3b67ce38fb2038d6ed05f63eed3558dbe2322213e +h100-aks-ubuntu-training-slurm: 808fd84a29508b063f6e4e429a52ef4a73a4c43aa8f3e14d7b135b41ba68bc89 +h100-any: 23ebf3f73f84d7dcfd1d7d9aa547d0b1a4a581c2d74b2220dde9c6644fc4d884 +h100-bcm-ubuntu-training: f8d62e665c5c6f7b4a790a2641b7edeb3d833f84ce458fd94fe65c4aa2bb7bc1 +h100-eks-ubuntu-inference-dynamo: 6563ca6ef7133b3db819960d0bf79fbfce6f8b8d4c8e948fa5a2b648efca8ca3 +h100-eks-ubuntu-inference-nim: fb4ee3446b811f9fa0fdd7847c209b83675e7862e215945568cd2b30ea1b2b02 +h100-eks-ubuntu-training-kubeflow: b9e71f097bdd633fc2c95c3fafe8bbb4b8e9b77b5f22fd4e9e272a4a23f1669d +h100-eks-ubuntu-training-slurm: 095acee397939ef639ec6914fd5fa02146740860337fe6c3ba4d82c9d077493b +h100-gke-cos-inference-dynamo: 5c5ad5b6c31348983f2891e16018f6cb88b90dcbba5aadaa65f17c0e13e4b034 +h100-gke-cos-training-kubeflow: 163e5739923552a1cc7c866434e8f50d4f64ab1a7711113bebbebf9a821766b4 +h100-gke-cos-training-slurm: 5ad98f0c0236eb6b3600a1470df8324bc03d77aea9044e8135f7277326b80e5f +h100-kind-inference-dynamo: 6a71cff45b7ad797099ba8844188ee103c9673f7b5864c9fdf6ce4ff5a6b907b +h100-kind-training-kubeflow: c5583a8463c01b360812aea83bf91397b9f7e2861d7a2abfbdefe125c9d22767 +h100-kind-training-slurm: 36a11fc1dee5cb554fe424b72fff08dc43a7a2a6349576aad38bc132ac166aea +h200-any: 3ecef0a8ba4812fe8eadb84dfeb82761a9a35b2a84ddba326c197e80c9da9d47 +h200-eks-inference: bc5548d165d48b59cee92dee47f27fe8fb1857107a09db95f01a563dda2f2255 +h200-eks-training: b780af133d538d0d0433aba70dd00763061745622b0aadd7dc9d01235e3de02e +l40s-any: 5c8ec41ec6423676c0dd58fb3d3c227f870ad3b62b06c267df4bb50d17525bd8 +l40s-oke-inference: de1b21f165ed75ffe30d97309dffecd241143e51a1eded44f78659b2a5b3f979 +l40s-oke-training: 00c344e4a2a5153052ef1ee0007a56add2c4d3f3a96e06dbbf29bd284d1f4b9f +monitoring-hpa: d608299f1380ac1eb39b17943d35807160ae4df14bbe8e4feba4b0c59b807c19 +ocp-inference-nim: ca3cfa5cc9a82326b30a73059752c017ba82d3d8944f14fcc0f25bdcd9e5de40 +ocp-training: 9c6f235c473271d07638074c603b517274f884b6a8e154acd22d5d5c00de9e93 +rtx-pro-6000-any: 633974116f00b36ef7b2ecb1c247abd1b197287acc7f8d362d56a9a7b1b9f5a4 +rtx-pro-6000-eks-ubuntu-inference-dynamo: 0b00a903d93a7797111e159bdf7517efd5b0f08bbdda4a27d64522a7af923720 +rtx-pro-6000-eks-ubuntu-inference-nim: 597cc3244844bee36cb342920fcb2436acd9b51014d392a1ca0bf9d87ec2f3bb +rtx-pro-6000-eks-ubuntu-training-kubeflow: b3ea07447897da22dca69ad26c692c108b6deb9ab6aa8bf013b80009d7cd8ce5 +rtx-pro-6000-lke-ubuntu-inference: ef2e3b1642cba6de918d2084412b8316a1e345a64d05f887a04d085c2cd21947 +rtx-pro-6000-lke-ubuntu-training: 4cd96eb0acaf6e7599716409cfd1b4fc3d00c160599e3773dd2598c248186839 diff --git a/pkg/recipe/testdata/catalog_parity_golden.yaml b/pkg/recipe/testdata/catalog_parity_golden.yaml index 06cffd0e0..4732e1680 100644 --- a/pkg/recipe/testdata/catalog_parity_golden.yaml +++ b/pkg/recipe/testdata/catalog_parity_golden.yaml @@ -6,11 +6,11 @@ a100-aks-ubuntu-training-kubeflow: 52a19136ff27701be3ebd3291d0838b405dd4200b9ea2ad1f15f5e9231afc164 a100-any: eedbde8040b6909da0e42d2af6345881ca5fcaacaa3ac83b222ba998c5c07c5e a100-eks-ubuntu-training-kubeflow: 9918500dfc098857fa643a8427d378ad205fc7914566095c8678c2b837b93e7a -a100-gke-cos-training-kubeflow: e2dedc04e012cf16d146c953bc1f6a58237afd305e1978b58976d67d5695cb37 +a100-gke-cos-training-kubeflow: 1071127f0f5ca3969479fc268bc2ff9d94c5e1f6453114bff38647cf5177759f a100-oke-ubuntu-training-kubeflow: c2f71c8a4bb59e2cca85fc007740eede1ded4a6514ec46071d7172f707c0fa13 b200-any: 2d0d453bcdcd89d5db6b0aab435aad942458b108a7b0060a48e73c88a96e11ca -b200-gke-cos-inference-dynamo: 37147bb9daf7782bd58349c97ef01fa72b70fb46f1e01746d8725869a83ecfc6 -b200-gke-cos-training-kubeflow: e219956eaff20825678d485dd656e6071a6a91afc180af9abe54122d93191164 +b200-gke-cos-inference-dynamo: 0744fd6b5d3c051ff9e1bdc1341077bb14a208ebac6c418fbdb38f2aa7c037d3 +b200-gke-cos-training-kubeflow: d79d47da234847bdc9cdd7d1613b3a5a3e28322a4f51419df2eb084717b05a55 bcm-inference: c608118b2f1c74e2d1d2d9defa90918f9da2cf6122afbf133d605f8a0154ea87 gb200-any: 548063fe33335835470c1bbf9fb9ace134061d06a8910409e68d90423377a41d gb200-eks-ubuntu-inference-dynamo: b15f4dd4cf5224ca76f7524f15611ed22910c4ee90334fe160594cb1592e9ecc @@ -30,9 +30,9 @@ h100-eks-ubuntu-inference-dynamo: 0b20a1afb8336d0fb7abe60dd5308cb2b76b96df2a4932 h100-eks-ubuntu-inference-nim: 00460f237a30b03854875ce52a9e7a58217878192cfeb3f52745b26d1ee66c18 h100-eks-ubuntu-training-kubeflow: 0940f0e8420302f91cd11f3d0451406de9259408245ef9410423ad09bb369766 h100-eks-ubuntu-training-slurm: 939f6422a3b79e68b5a43225a838ecba9d2878d57cd9b4a98207d6d2860b1193 -h100-gke-cos-inference-dynamo: 904aeab7e68bce844fc9a72a9a79f27d28f77d044d510fdcbaecce596cefe983 -h100-gke-cos-training-kubeflow: 2ec3f85e22a551c48bca2a67af7211965c405f7a6fe7e4fa8be3a161311943a9 -h100-gke-cos-training-slurm: b74e224e7362f803da02ed60e090954d3c70d964cdc6db62df4bac4b5a060f70 +h100-gke-cos-inference-dynamo: 4093ad304575e93f67f1b7ef91f8e1641637b80159fccb9e3ea4a2f010da62cb +h100-gke-cos-training-kubeflow: 32647468b25ac6f336a85b7d9cda4e080a5283d8b3781700d81864e2f92af4cf +h100-gke-cos-training-slurm: 73861db80be53fbe09a97bd3aeb574a1bfdc0a7669796077cfcc2330aab9a25b h100-kind-inference-dynamo: b17a7431bf9b5cd97211d343274324bdf7871b44b504f77bd575f54092c993db h100-kind-training-kubeflow: 49f22bb0ba0b62c8c728cb801e99af9d4515ecbff4151100a0788bd3f4c1dcf2 h100-kind-training-slurm: 891a37a0ff30b83bae3252eaae8305fb1fc4f29ddd3cafcb751f561308b738a7 From f9c9826c763243018cf0dd8f5ae99a7920765c95 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Tue, 1 Sep 2026 09:14:36 -0700 Subject: [PATCH 14/16] docs(demos): align demo output with the held 580.173.02 driver pin The demos were updated to the v26.7.0 chart default 595.91.07 while the recipe deliberately holds driver.version at 580.173.02, so every demo output line showed a value the tool will not print. Three defects: - demos/query.md and demos/dynamic.md documented 595.91.07 as rendered output. A user running the documented commands gets 580.173.02. - demos/dynamic.md's step-5 check 'grep -c version: 595 values.yaml # 0' became vacuous: no bundle contains 595 with the pin at 580, so it prints 0 whether or not --dynamic actually moved the value. Restored to 'version: 580', which is what makes the zero meaningful. - demos/query.md rendered kernelModuleType after rdma in an alphabetically sorted subtree. Moved between enabled and maxParallelUpgrades. demos/** is not executed in CI (tools/coverage/model.go:46), so nothing would have caught this. Signed-off-by: Yuan Chen --- demos/dynamic.md | 8 ++++---- demos/query.md | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/demos/dynamic.md b/demos/dynamic.md index f2c1be1b1..8c5caf331 100644 --- a/demos/dynamic.md +++ b/demos/dynamic.md @@ -98,7 +98,7 @@ cat ./bundle/010-gpu-operator/cluster-values.yaml # # Generated by AICR # --- # driver: -# version: 595.91.07 +# version: 580.173.02 ``` And `install.sh` applies it after the baked values, so it wins: @@ -116,12 +116,12 @@ Three quick checks, the same ones the chainsaw suite enforces: ```shell # a) gone from the baked values.yaml (no match) -grep -c 'version: 595' ./bundle/010-gpu-operator/values.yaml +grep -c 'version: 580' ./bundle/010-gpu-operator/values.yaml # 0 # b) present in cluster-values.yaml grep version ./bundle/010-gpu-operator/cluster-values.yaml -# version: 595.91.07 +# version: 580.173.02 # c) install.sh references cluster-values.yaml grep -q cluster-values.yaml ./bundle/010-gpu-operator/install.sh && echo wired @@ -193,7 +193,7 @@ rm -rf ./b-oci oras pull "$REF" -o ./b-oci (cd ./b-oci && aicr verify .) cat ./b-oci/010-gpu-operator/cluster-values.yaml -# version: 595.91.07 +# version: 580.173.02 ``` > ttl.sh is public; anything pushed is world-readable until it expires. Fine for a throwaway demo bundle, not for anything sensitive — push to your own registry by pointing `spec.bundle.output.target` (or `--output`) at it, e.g. `oci://registry.example.com/aicr-bundle:tag`. Transport is separate: `spec.bundle.registry.plainHTTP: true` only switches HTTP instead of HTTPS (enable it only when the registry is intentionally served over plain HTTP, such as a local test registry); leave it off for any HTTPS registry, and use `spec.bundle.registry.insecureTLS: true` to skip TLS verification against a self-signed HTTPS registry. diff --git a/demos/query.md b/demos/query.md index 3bda0bff9..32c6f7ede 100644 --- a/demos/query.md +++ b/demos/query.md @@ -38,7 +38,7 @@ aicr query \ --selector components.gpu-operator.values.driver.version ``` -> `595.91.07` +> `580.173.02` Subtree — full driver block: @@ -50,11 +50,11 @@ aicr query \ ```yaml enabled: true +kernelModuleType: auto maxParallelUpgrades: 5 rdma: enabled: false -kernelModuleType: auto -version: 595.91.07 +version: 580.173.02 ``` ## Differentiation: Same Selector, Different Criteria From ada649847166d837afa5ea056bf03fd5229f3423 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Wed, 2 Sep 2026 18:22:24 -0700 Subject: [PATCH 15/16] test: re-audit DRA chart K8s floor and goldens after rebase onto main origin/main now includes #2449 (raise K8s floors to clear the DRA chart's kubeVersion), which this branch's earlier golden-regeneration commits predate. Rebasing surfaced two gaps: - auditedDRAChartFloors in dra_k8s_floor_test.go was still pinned to DRA driver 0.4.1; this branch bumps to 0.5.0. Verified kubeVersion is unchanged (>=1.32.0-0, minor 32) directly against the published dra-driver-nvidia-gpu 0.5.0 chart, so only the audited version string moves, not the floor itself. - catalog_parity_golden.yaml and stock_render_golden.yaml needed a fresh AICR_UPDATE_GOLDEN=1 regeneration against the combined state (this branch + main's intervening changes), superseding the rebase's mechanical conflict resolution. Signed-off-by: Yuan Chen --- pkg/bundler/testdata/stock_render_golden.yaml | 97 ++++++++++--------- pkg/recipe/dra_k8s_floor_test.go | 6 +- .../testdata/catalog_parity_golden.yaml | 97 ++++++++++--------- 3 files changed, 101 insertions(+), 99 deletions(-) diff --git a/pkg/bundler/testdata/stock_render_golden.yaml b/pkg/bundler/testdata/stock_render_golden.yaml index 105900fac..17d4954c6 100644 --- a/pkg/bundler/testdata/stock_render_golden.yaml +++ b/pkg/bundler/testdata/stock_render_golden.yaml @@ -3,51 +3,52 @@ # # One entry per leaf overlay: a digest over its fully rendered helm-deployer # bundle tree (sorted relative paths paired with per-file content hashes). -a100-aks-ubuntu-training-kubeflow: 611e7dd4b638025c5ccc14bbec5afc24eda359b52f6ab82b0db0ce0235c80fa9 -a100-any: f3d55941c7fe2549a47ec14459773d26372488c6f8e4747fdf2682865535502c -a100-eks-ubuntu-training-kubeflow: f9a3471fcfb2a61e73f72767308a878a9deaba964603bda7a67445c033a93428 -a100-gke-cos-training-kubeflow: 4174ba7a47f024ff4f20918aea6d2ab237dc9e4c83e6bdf810259d0ca18d6f94 -a100-oke-ubuntu-training-kubeflow: ec7553d5d5c9884b75d626afbe2cfdc11891f417a5b8aabc49615eba66c1e0cb -b200-any: e4f8e29a45fdde19127181030eac2b0e97eb28c2e9d14fd28e30f62e31a6da54 -b200-gke-cos-inference-dynamo: 6cd7fd31f443982be752cc433e4143eeecd55a24fd0975ae1100eebc0df2c7fa -b200-gke-cos-training-kubeflow: fca1a9a7699f0811b3833b042e6068be759f366b29c1f4efcb2a63267cb04d9c -bcm-inference: aeadd0f0e6733f84cdacaad3f5d0610b45463aa049370a40b4543dcd6f7ece30 -gb200-any: 8355e5f6279cf96463c58ab5ced5237020742a9580fadff714da282489201690 -gb200-eks-ubuntu-inference-dynamo: c5ae52dbcc96704c07f5210c6fb604605651d08244a99e221b03232723f760e0 -gb200-eks-ubuntu-training-kubeflow: 67a7077f82ac9453c6c554e19cdba108154ad12bf94e41aaa7b81aceda1fcdb7 -gb200-eks-ubuntu-training-slurm: 28ad1fa1284dcd80f2547b0ad7f60d459cf47bfc9257bdfd9183dae50f9c79ab -gb200-oke-ubuntu-inference-dynamo: 4c816f06e392f17e6739cfaca9f72ad1200f6844eff593b98db7041ed02e55fe -gb200-oke-ubuntu-training-kubeflow: 37a4b534cd064726ce90841f2a5c1dd60521b930f50375d843a01370aa1476ab -gb300-any: 10258903d47723de166268d12c45e7f7587b3604ee9c1c63f853626615b0f18f -gb300-eks-ubuntu-inference-dynamo: cab5f3ba54cfb754534c8e7cbebe772d7fdd8a382bc4400615598c8abc4625eb -gb300-eks-ubuntu-training-kubeflow: 76ac339521a7a11ead88ee8c22e59cceb438990ea408c47ccafa2c3c61480506 -h100-aks-ubuntu-inference-dynamo: 14ea86cb775a62a4010ca3a003fd2cdbf2bbdc50184db5418d1458e126da5f34 -h100-aks-ubuntu-training-kubeflow: eafd5a02b9b87f5c543a60a3b67ce38fb2038d6ed05f63eed3558dbe2322213e -h100-aks-ubuntu-training-slurm: 808fd84a29508b063f6e4e429a52ef4a73a4c43aa8f3e14d7b135b41ba68bc89 -h100-any: 23ebf3f73f84d7dcfd1d7d9aa547d0b1a4a581c2d74b2220dde9c6644fc4d884 -h100-bcm-ubuntu-training: f8d62e665c5c6f7b4a790a2641b7edeb3d833f84ce458fd94fe65c4aa2bb7bc1 -h100-eks-ubuntu-inference-dynamo: 6563ca6ef7133b3db819960d0bf79fbfce6f8b8d4c8e948fa5a2b648efca8ca3 -h100-eks-ubuntu-inference-nim: fb4ee3446b811f9fa0fdd7847c209b83675e7862e215945568cd2b30ea1b2b02 -h100-eks-ubuntu-training-kubeflow: b9e71f097bdd633fc2c95c3fafe8bbb4b8e9b77b5f22fd4e9e272a4a23f1669d -h100-eks-ubuntu-training-slurm: 095acee397939ef639ec6914fd5fa02146740860337fe6c3ba4d82c9d077493b -h100-gke-cos-inference-dynamo: 5c5ad5b6c31348983f2891e16018f6cb88b90dcbba5aadaa65f17c0e13e4b034 -h100-gke-cos-training-kubeflow: 163e5739923552a1cc7c866434e8f50d4f64ab1a7711113bebbebf9a821766b4 -h100-gke-cos-training-slurm: 5ad98f0c0236eb6b3600a1470df8324bc03d77aea9044e8135f7277326b80e5f -h100-kind-inference-dynamo: 6a71cff45b7ad797099ba8844188ee103c9673f7b5864c9fdf6ce4ff5a6b907b -h100-kind-training-kubeflow: c5583a8463c01b360812aea83bf91397b9f7e2861d7a2abfbdefe125c9d22767 -h100-kind-training-slurm: 36a11fc1dee5cb554fe424b72fff08dc43a7a2a6349576aad38bc132ac166aea -h200-any: 3ecef0a8ba4812fe8eadb84dfeb82761a9a35b2a84ddba326c197e80c9da9d47 -h200-eks-inference: bc5548d165d48b59cee92dee47f27fe8fb1857107a09db95f01a563dda2f2255 -h200-eks-training: b780af133d538d0d0433aba70dd00763061745622b0aadd7dc9d01235e3de02e -l40s-any: 5c8ec41ec6423676c0dd58fb3d3c227f870ad3b62b06c267df4bb50d17525bd8 -l40s-oke-inference: de1b21f165ed75ffe30d97309dffecd241143e51a1eded44f78659b2a5b3f979 -l40s-oke-training: 00c344e4a2a5153052ef1ee0007a56add2c4d3f3a96e06dbbf29bd284d1f4b9f -monitoring-hpa: d608299f1380ac1eb39b17943d35807160ae4df14bbe8e4feba4b0c59b807c19 -ocp-inference-nim: ca3cfa5cc9a82326b30a73059752c017ba82d3d8944f14fcc0f25bdcd9e5de40 -ocp-training: 9c6f235c473271d07638074c603b517274f884b6a8e154acd22d5d5c00de9e93 -rtx-pro-6000-any: 633974116f00b36ef7b2ecb1c247abd1b197287acc7f8d362d56a9a7b1b9f5a4 -rtx-pro-6000-eks-ubuntu-inference-dynamo: 0b00a903d93a7797111e159bdf7517efd5b0f08bbdda4a27d64522a7af923720 -rtx-pro-6000-eks-ubuntu-inference-nim: 597cc3244844bee36cb342920fcb2436acd9b51014d392a1ca0bf9d87ec2f3bb -rtx-pro-6000-eks-ubuntu-training-kubeflow: b3ea07447897da22dca69ad26c692c108b6deb9ab6aa8bf013b80009d7cd8ce5 -rtx-pro-6000-lke-ubuntu-inference: ef2e3b1642cba6de918d2084412b8316a1e345a64d05f887a04d085c2cd21947 -rtx-pro-6000-lke-ubuntu-training: 4cd96eb0acaf6e7599716409cfd1b4fc3d00c160599e3773dd2598c248186839 +a100-aks-ubuntu-training-kubeflow: 88974161cf87092faee1fd5ed39f6ef03ec217a630ee10ae34bee027dd8c97a4 +a100-any: 2cbbcd65973e54c164e8c0025142c43b0384e913c3ade2ebc4b3580e90270b17 +a100-eks-ubuntu-training-kubeflow: 66a2c9fc3172d79e1710a7522cca2c683b188193ddfc58d1b1953417b5354362 +a100-gke-cos-training-kubeflow: 34a990b268eac35a9fd00010563ef1bd964525bbbe83121b7aeeb54c7ba02954 +a100-oke-ubuntu-training-kubeflow: 5b79e3472e93ada835a7ec85b038857a1db3f4bf7df34033388b8c0c867a654e +b200-any: 96ce9531ff9d35b6f92577906e5c6a76a0df0648eb6ff0b338c7c92ae555b536 +b200-gke-cos-inference-dynamo: c635c062c09d8ea9cd0c445af1cca146ecdfdbd0fa9c1be9822f592aa0159d7f +b200-gke-cos-training-kubeflow: de66551f631b79451dd86ba10479748dcf7a911cc70ae538f3fdea5b0a8dd642 +bcm-inference: d7fe93b0a299ae87b15cf7fbb81a78b72b4d868a25c6fcbb70aa26f97a31f548 +gb200-any: e96732f58ce13d20e6b355afa20346bc25d88a9197d303064bdb5de91404279a +gb200-eks-ubuntu-inference-dynamo: 668a1862791c5d41923f66ef263603b63f56d384e95e1579049cb4fc793d96ee +gb200-eks-ubuntu-training-kubeflow: b13b8d7ac8e45fedf00df2eb39699ddcdb5bd4c7bc94478c465aa582deb0b769 +gb200-eks-ubuntu-training-slurm: 03cac6712a2749ca6bb75f472544659c6a95d408bbddc56ac94aef5dc03c335a +gb200-oke-ubuntu-inference-dynamo: d65471fc0ace996a5b46b20c884b071ca33080c4f98b22d3fcfa59ff9770e9fb +gb200-oke-ubuntu-training-kubeflow: 905229e58cd9920e1b4ae57405518d482e3d098f1b01aa27509d9447cb6b2f55 +gb300-any: 51e94d57f8bf60b2d2920c8255a9933e2f1fcb8c8a594793c7666173154b03fe +gb300-eks-ubuntu-inference-dynamo: 172200808700cc9747dd372f61b2cf32bd2b5ab8e7ddf2ebbcb23c81f22603ed +gb300-eks-ubuntu-training-kubeflow: d187e3797e92b203c361518b3fbc5af76acd0a62071bcf3d3fa2c093cd94f027 +h100-aks-ubuntu-inference-dynamo: 7830de226b961b7d3c3165dc559c2669d429939b800a87a54b8427bc157dc854 +h100-aks-ubuntu-training-kubeflow: 30caffd8764966e7432672b1dcd74861012beaaf5ae0aeb32cd67ebca0c102af +h100-aks-ubuntu-training-slurm: b1cc76701977baab3aca432f0022f8c0842c547058cf3fbbad261db3b1c068b3 +h100-any: 63ec1b86ddeee974b2da15b1f8eafed9efab968a472b09d6179e1d117e667bc2 +h100-bcm-ubuntu-training: 657509ca168955fd6624efe6c7abab91040df4ac7b0aedf8a7ed4f5e7d00caa4 +h100-eks-ubuntu-inference-dynamo: 7d16e73b8ce23edbccfc8f3ddc2d5dc000c5e6813240f157f1f8230efabe8d4e +h100-eks-ubuntu-inference-nim: 0b3525d5cf7b471e27556d7b68f05e5b8c606d90a62cbf5d2b6dbeeb4aeff67d +h100-eks-ubuntu-training-kubeflow: f031ab2be281a5401cd3426fd5b8cb17020944938f38ddeaeb63540c96c149f1 +h100-eks-ubuntu-training-slurm: ff33f128fa254c9057c672a3c9b98452b7a648f20284ada0479446ca2b2b1f8f +h100-gke-cos-inference-dynamo: eebc1ae69635ed9e4732adf88ec81507c4ea23fd3875084d7943a6710b028e0c +h100-gke-cos-training-kubeflow: f762c480f098f1f936ce7ab749f372fcfebf5d4ee11d06dbd473449bbb530e9a +h100-gke-cos-training-slurm: 28697b311b2d218e5bc88499720ae21f592bf751ec55b1f2975b9d7c37427b9f +h100-kind-inference-dynamo: 2a7160eb1d0b494fd0b8dd7a89496ac11969f3ab35969264fd83d588dc70def0 +h100-kind-training-kubeflow: e77c4afe346b00d66a7329b5d74dc655c0219f15514dcbf58007c8db27564de3 +h100-kind-training-slurm: 76f993dd8d5e6dc113e38e98b01d99c1fa26157093e808345b09dfb5410a5059 +h200-any: d852f8915d79b5a8e527116f2aeede59df1d2d68969206484f40184afa8f5ce9 +h200-eks-inference: 479867958a103efcdb32b4592a2e1e1ca54f512ff855449ffbc8bd1e9dddb471 +h200-eks-training: 4078ec6949fe054f913665542b3256f103d8b5630855b61beea6568ac63a108a +l40-any: c386fac8b86684354177fb760c6cd1d41316d46d9a8438a1d817c82565aa013f +l40s-any: f493cca8f86fa263654bda25caeb840430f95b3713eac0b67426de1b20ec4f9b +l40s-oke-inference: 46d6020c6ce89505109ae2eb091789e58a5e5bbeb829c97e1a5e80c7d4d8302f +l40s-oke-training: 654cf1f167e7eebe6e6ddd4cd85a978f3170ad7c0c9a8a49bd853ad44990885c +monitoring-hpa: b86b773c621d5eca9fafc1e12750427111c0865a762496ce6aba28cf4ee4aa7a +ocp-inference-nim: bb3c7fa78c972241b47abedbff3069d4fad35ab0c64d5f0172c633d8ed610b27 +ocp-training: 1f457d2c2aed921c2a3281cca1a55b1181d979cc99f07c1b51b9ce827379b401 +rtx-pro-6000-any: ad8b509fb7f6c25c765d3b9112b03594e3aac99a48a9e6c11df790b131488284 +rtx-pro-6000-eks-ubuntu-inference-dynamo: bedbdf1b531f3bb8de74e38ff4df094297a572698dffa121aa3ee91d5918c942 +rtx-pro-6000-eks-ubuntu-inference-nim: 59cfc17d0cfa2429e637cf3cecfa2e1d397c8d3d2dc7f11c6ec5995d392a5382 +rtx-pro-6000-eks-ubuntu-training-kubeflow: 6b1e4c1b3e6eaab010dd12ed72f7374e6e206ebb065a437ec8ca56518e282f32 +rtx-pro-6000-lke-ubuntu-inference: b207bf93e21bcad9c3d2e4b59dd6696cd12e8967e7a5e9c7ba6fce883e7943c0 +rtx-pro-6000-lke-ubuntu-training: 2710efe3081974b752f51a883a99e063071abfa1038a3a49e76cbac711b8ca00 diff --git a/pkg/recipe/dra_k8s_floor_test.go b/pkg/recipe/dra_k8s_floor_test.go index 5baaf2056..c08bf8c55 100644 --- a/pkg/recipe/dra_k8s_floor_test.go +++ b/pkg/recipe/dra_k8s_floor_test.go @@ -52,9 +52,9 @@ var auditedDRAChartFloors = map[string]struct { minor int }{ // oci://registry.k8s.io/dra-driver-nvidia/charts/dra-driver-nvidia-gpu - // kubeVersion: ">=1.32.0-0" - "nvidia-dra-driver-gpu": {version: "0.4.1", minor: 32}, - "nvidia-dra-driver-gpu-ocp": {version: "0.4.1", minor: 32}, + // kubeVersion: ">=1.32.0-0" (unchanged from 0.4.1 to 0.5.0) + "nvidia-dra-driver-gpu": {version: "0.5.0", minor: 32}, + "nvidia-dra-driver-gpu-ocp": {version: "0.5.0", minor: 32}, } // draChartKubeVersionMinor is the highest audited floor across the enrolled DRA diff --git a/pkg/recipe/testdata/catalog_parity_golden.yaml b/pkg/recipe/testdata/catalog_parity_golden.yaml index 4732e1680..ccc316a70 100644 --- a/pkg/recipe/testdata/catalog_parity_golden.yaml +++ b/pkg/recipe/testdata/catalog_parity_golden.yaml @@ -3,51 +3,52 @@ # # One entry per leaf overlay: sha256 of its deterministically-marshalled # resolved recipe. A moved digest means that recipe's resolved bytes changed. -a100-aks-ubuntu-training-kubeflow: 52a19136ff27701be3ebd3291d0838b405dd4200b9ea2ad1f15f5e9231afc164 -a100-any: eedbde8040b6909da0e42d2af6345881ca5fcaacaa3ac83b222ba998c5c07c5e -a100-eks-ubuntu-training-kubeflow: 9918500dfc098857fa643a8427d378ad205fc7914566095c8678c2b837b93e7a -a100-gke-cos-training-kubeflow: 1071127f0f5ca3969479fc268bc2ff9d94c5e1f6453114bff38647cf5177759f -a100-oke-ubuntu-training-kubeflow: c2f71c8a4bb59e2cca85fc007740eede1ded4a6514ec46071d7172f707c0fa13 -b200-any: 2d0d453bcdcd89d5db6b0aab435aad942458b108a7b0060a48e73c88a96e11ca -b200-gke-cos-inference-dynamo: 0744fd6b5d3c051ff9e1bdc1341077bb14a208ebac6c418fbdb38f2aa7c037d3 -b200-gke-cos-training-kubeflow: d79d47da234847bdc9cdd7d1613b3a5a3e28322a4f51419df2eb084717b05a55 -bcm-inference: c608118b2f1c74e2d1d2d9defa90918f9da2cf6122afbf133d605f8a0154ea87 -gb200-any: 548063fe33335835470c1bbf9fb9ace134061d06a8910409e68d90423377a41d -gb200-eks-ubuntu-inference-dynamo: b15f4dd4cf5224ca76f7524f15611ed22910c4ee90334fe160594cb1592e9ecc -gb200-eks-ubuntu-training-kubeflow: fc6b234dd59ec815ba4bc67c695ff04057dbf3a233ba68662c54b96450ba035b -gb200-eks-ubuntu-training-slurm: 4b2007f13eb5c953620a3e18271e3f5b8d8a407258a36c1c47882012490d0470 -gb200-oke-ubuntu-inference-dynamo: 4ce2e85e13d125948e91614f0d07ebb70968ea0d93beec9feba03863f420f282 -gb200-oke-ubuntu-training-kubeflow: 3f0394a1b85b3d16147a31ecf55088d899c3b103ead7cb2f8a25183cb3a1575d -gb300-any: 10eb8b5cc815ca6a190412e8e344d261c0153cc65e2bf8266d89f6d821171da3 -gb300-eks-ubuntu-inference-dynamo: 9a4d41b7e6c57ec6cb44df5f6af64ca3f4d15a4568d6dfaaa16dfc0e5b791877 -gb300-eks-ubuntu-training-kubeflow: 460cb834803767f98f70f6bad7e006d22ebd79f83aa4c62f3feace0f896bd579 -h100-aks-ubuntu-inference-dynamo: e16910a1555ed867d5c85b63d6bea4861e585f1c4cbffeb81b9315fe04feb551 -h100-aks-ubuntu-training-kubeflow: 0ef4f87a06b668338e8416eb7b85e5ac63f472ebbd478c69db1a6e36d9cdac6a -h100-aks-ubuntu-training-slurm: f43f166e8d0af0447fa581e227fff3088d8a0d5efb9bac8cda4aff2f6cf84d04 -h100-any: 879ed75bd46ef020cc9f31eb03bceb49debe3719c2dcf567d072ac5b13ba3d2e -h100-bcm-ubuntu-training: 2d25d1fa54d969088215f4f95d2eabd2aead572a4223d2421f6d9ffaaf15be61 -h100-eks-ubuntu-inference-dynamo: 0b20a1afb8336d0fb7abe60dd5308cb2b76b96df2a4932fe18196468a08b1012 -h100-eks-ubuntu-inference-nim: 00460f237a30b03854875ce52a9e7a58217878192cfeb3f52745b26d1ee66c18 -h100-eks-ubuntu-training-kubeflow: 0940f0e8420302f91cd11f3d0451406de9259408245ef9410423ad09bb369766 -h100-eks-ubuntu-training-slurm: 939f6422a3b79e68b5a43225a838ecba9d2878d57cd9b4a98207d6d2860b1193 -h100-gke-cos-inference-dynamo: 4093ad304575e93f67f1b7ef91f8e1641637b80159fccb9e3ea4a2f010da62cb -h100-gke-cos-training-kubeflow: 32647468b25ac6f336a85b7d9cda4e080a5283d8b3781700d81864e2f92af4cf -h100-gke-cos-training-slurm: 73861db80be53fbe09a97bd3aeb574a1bfdc0a7669796077cfcc2330aab9a25b -h100-kind-inference-dynamo: b17a7431bf9b5cd97211d343274324bdf7871b44b504f77bd575f54092c993db -h100-kind-training-kubeflow: 49f22bb0ba0b62c8c728cb801e99af9d4515ecbff4151100a0788bd3f4c1dcf2 -h100-kind-training-slurm: 891a37a0ff30b83bae3252eaae8305fb1fc4f29ddd3cafcb751f561308b738a7 -h200-any: 959608a2eb8060c6df2ef7caf453a1500f1adc8790b8c228851c603d15b6e908 -h200-eks-inference: dd2814a49106bfae9623bd310587ae4bcf378fba20039726796272fe23a571cc -h200-eks-training: cd7439667773a77b523a7337e672f4b8a9828bf88af618436094dc556746acb1 -l40s-any: 83b2b92d8b43e8f23754ff9608f02a6d64d7de28f8ccb3b7d6ae67679cffac93 -l40s-oke-inference: 9761d9172ba2c7561e0406392e2c0828450f6e3d5537c61bdaf5732d80fca487 -l40s-oke-training: 4b40b25e4a5620d3cea304df67209b397e3813190ee2ff4c7dd5798690ecd10d -monitoring-hpa: 7156deeffed71f32316b4a2501c771033d67dc2de3d3181311e985cc3625e557 -ocp-inference-nim: 08f9f5d03f030160eb6951bd238c24a400c41d62dcf721589abb3d265df8e3fc -ocp-training: 3998c76724c4189cc463e5cf473a3cd97d8c0eaf3299336be7f1ffac9cb11e6f -rtx-pro-6000-any: 62760c1f1e8287ae8c37b20d306546469bed84a742ea2b143aad52f942575f7e -rtx-pro-6000-eks-ubuntu-inference-dynamo: 2e14248538447738e7ea1699b0c0730dfd1e18b8aa8588ab0540903874e3bc2e -rtx-pro-6000-eks-ubuntu-inference-nim: b1c918b05e42118edcb51a164a9e251c944243d3e8d0127855ec361f8740ee52 -rtx-pro-6000-eks-ubuntu-training-kubeflow: cb833e1bcc917f105a106f4e36186ee55c665c641c22bd0a5da4206284b04daf -rtx-pro-6000-lke-ubuntu-inference: 1de10952036333b9f1d475b5ae212d54e4ed226ff3a44c17b13415bb708e5368 -rtx-pro-6000-lke-ubuntu-training: 296c4559de7977f238303e787407e57489a0dae8b24249f9f416a0425d4d48c7 +a100-aks-ubuntu-training-kubeflow: 1cc3ce4d52a71b6137179afa51859e149b656e71c32c89c8be23f193f7d28ed2 +a100-any: 5becb0645a3c787027e459adbcd093c8656f6c1e9370c90f9827e4978bca580c +a100-eks-ubuntu-training-kubeflow: a743c24bf62271a407b6608dd6099113ce4baaabe9506b016df6a3088e0c80c7 +a100-gke-cos-training-kubeflow: a8f9afa0cf6045d7789d1385b5cdabe654170da00a3894de446c34100dd60acb +a100-oke-ubuntu-training-kubeflow: 2bc755fc494c4db63788544b7bad2c98dfc0b4322a1cbe65364993fb40e38db6 +b200-any: 74665d86b2781705d2ecc7e28a7c45ecac931560095bacdd48cfcc5249714ac0 +b200-gke-cos-inference-dynamo: 72835241a6465ce0edb61333a18f34a97a48ce1e180d8bdb695755ede74de105 +b200-gke-cos-training-kubeflow: 5d8da216a34ed3935c2b0f0732f3bcf1187081585bce4f28610330a03e7b7ab6 +bcm-inference: fbad7e9704b9fbcd099ce9ea0799cecb0a0eb0fbc46f8b758ea5c1299be71db4 +gb200-any: 55cd7d2a2d33f677f10d58a1ddfbd02b79026e7a959d19d244c229437f8d8b69 +gb200-eks-ubuntu-inference-dynamo: 77cc897bd4b610cb0bf40f7b0ee1063c83b1af5065f5bbdad5ef026679135ce1 +gb200-eks-ubuntu-training-kubeflow: 8510ee4d66330a42add297565d531c8daa5d525006b0afa1e07dd442c72c8f36 +gb200-eks-ubuntu-training-slurm: ecafb0d201f905a1a09556394cec2bdc601de3c556e9c7502bc98d998d3fe17e +gb200-oke-ubuntu-inference-dynamo: d99cb6ca728d1ff0f4c56e4986d3b8361c280b5de526c3651614d663d7d91e14 +gb200-oke-ubuntu-training-kubeflow: 21b58a301871cffdd40de2c10fd4f6ec55833a9621a5eebf864816ed941d8adb +gb300-any: e46a40fd6441b6e1348d8b9bcb60ef78623e22980f2af75b466b7aaa2b6a9960 +gb300-eks-ubuntu-inference-dynamo: b228697b4c1e5040a853b602b6c41dcb6bfd7a7ba002b7bddbe688388a809c21 +gb300-eks-ubuntu-training-kubeflow: 9e04ac26502c5e3968a08257062ac01fd53d28962c408b72e68f6a95db22ea8f +h100-aks-ubuntu-inference-dynamo: 85ccdf260249a67a5a1ff965fd0226e053bd332c4ec5bbac22e1e2c006b2da0c +h100-aks-ubuntu-training-kubeflow: a8ad56b9c749ec8cc84d3c866ed8520cf27bc364a067938101ad635633071dc2 +h100-aks-ubuntu-training-slurm: 9791558c8e0e44b8462511d5ef03fa21c26bf0a3993fe17d11c4d9e8c4420605 +h100-any: 887124130cfc77b01a0c3dc8385782de13b4ad359314e607a6034443f803884c +h100-bcm-ubuntu-training: 790117ef6e991a27720954877e6f9e0e6d54b6f32c3988a22c388c104663c8ae +h100-eks-ubuntu-inference-dynamo: 1a49679c9fc814596c2cdbcccba7baf53409d53bc21195340bfb4d21ab2c2d51 +h100-eks-ubuntu-inference-nim: 6db15a210537c5695abbf7ebb57c2cfc33c4cab97ce9b0007d366bdc1bda33e6 +h100-eks-ubuntu-training-kubeflow: 677de790e9697c102e850c2394d521fdba4ba6853004f196c65f59abba051ea6 +h100-eks-ubuntu-training-slurm: 85ad4d6705e0ea7bc73d91fb9a1e99e5026adbe9dc6a322ac0bb3187fbb1ac78 +h100-gke-cos-inference-dynamo: 476f02ba3f72a6b81d1adfb01ca4acb1b47982028e03304f41e12c9df1b0d446 +h100-gke-cos-training-kubeflow: bc98f3822cca567c496a91441919089f0d260a2f8cb8fedf6c678a89bf03cfb7 +h100-gke-cos-training-slurm: f1aad8397ac49789794875ca0836189be018c120c98ac718edf9f5fd90356b0c +h100-kind-inference-dynamo: 33959156bba4e30c4a1336e2f1d79e10eb0de1946847f70b0c1090201a610f6b +h100-kind-training-kubeflow: b5dfa79c42619c5b5995f932af56f343e4d0588344e8d37d36cab7483d1cee03 +h100-kind-training-slurm: 8549b91c859978283d7371666565273a1c75e2908d9167ccaafa75f019b4d6de +h200-any: d8cce8f74339053cb6c3ce33d99e12012f73b8c675a77403b42d6e68d2a5d53d +h200-eks-inference: 0232aa2a3399cfd7faf04950fdb0136df4174c79a5999814a4dc5f8b57ac0893 +h200-eks-training: 5ca88ad1c13f778513cc972f28ff38736d266e6187d87d2d30ece8e6013353fe +l40-any: 9d5abac3c21c8bca2e4d8b2015691dc3f4f834db4590643c7a5a55ac0398fc35 +l40s-any: 79a8c1afe261c6e214e1f3019c215625c588192a0af917e6c5ab0049edd7d4df +l40s-oke-inference: 3c8d3fb233f8ee8d608c4731c5dad38d9237cc04960b2e46051313062a1ad267 +l40s-oke-training: 58c317707e06fd78ff328f4f420fba5963b1e89962907717b10d1da2980ba454 +monitoring-hpa: 8206ceb11958dadac846292bb693e7757762528d99d00316e4aa427ec4cb3ea9 +ocp-inference-nim: f1f99e6e99b4f7084ab4cba65e38dddbefb9d7bcf9579f16c86d5f76508b232f +ocp-training: 4c5d4e277307f9f08c7ed5a47d79e61b7a8bbc5970ba6554a0a4471b4e7d405e +rtx-pro-6000-any: c54af648defc635cb8540064f4eb1cf605e4a95dd3f221a089cdfcc29d409438 +rtx-pro-6000-eks-ubuntu-inference-dynamo: 6448f2b42eec2f52df07bee010ab433c96daf2f592034bb07dcf6519b365db12 +rtx-pro-6000-eks-ubuntu-inference-nim: a63f7ae7621c251b49f3aeec1b4f18add34e29206f2d33a804694a43e87c1fb9 +rtx-pro-6000-eks-ubuntu-training-kubeflow: 95a0fd3483bf06446c640add474f260d7f0691f2049579f251871b466a204e1e +rtx-pro-6000-lke-ubuntu-inference: fe72e435418de60273524fb8b502a85c727f90adc1603cde8456b8914bec0bd9 +rtx-pro-6000-lke-ubuntu-training: e48c0b2459d94cfa4e0cb487290762dc5ba76f46a95323d5421dab569358212e From 18c1d634749ac72c4bcd21ab2ec8b3f7a3dcf9a7 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Thu, 3 Sep 2026 07:57:57 -0700 Subject: [PATCH 16/16] test: regenerate goldens after rebase onto main Catch-up rebase onto main (now carrying #2547, #2549 and the RTX PRO 6000 / dynamo-platform work). The two parity goldens conflict on every such rebase because both this branch and main regenerate them; resolved by regenerating fresh against the combined state rather than taking either side's bytes. Signed-off-by: Yuan Chen --- pkg/bundler/testdata/stock_render_golden.yaml | 26 +++++++++---------- .../testdata/catalog_parity_golden.yaml | 26 +++++++++---------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/pkg/bundler/testdata/stock_render_golden.yaml b/pkg/bundler/testdata/stock_render_golden.yaml index 17d4954c6..71757c70d 100644 --- a/pkg/bundler/testdata/stock_render_golden.yaml +++ b/pkg/bundler/testdata/stock_render_golden.yaml @@ -9,31 +9,31 @@ a100-eks-ubuntu-training-kubeflow: 66a2c9fc3172d79e1710a7522cca2c683b188193ddfc5 a100-gke-cos-training-kubeflow: 34a990b268eac35a9fd00010563ef1bd964525bbbe83121b7aeeb54c7ba02954 a100-oke-ubuntu-training-kubeflow: 5b79e3472e93ada835a7ec85b038857a1db3f4bf7df34033388b8c0c867a654e b200-any: 96ce9531ff9d35b6f92577906e5c6a76a0df0648eb6ff0b338c7c92ae555b536 -b200-gke-cos-inference-dynamo: c635c062c09d8ea9cd0c445af1cca146ecdfdbd0fa9c1be9822f592aa0159d7f +b200-gke-cos-inference-dynamo: 9a257ac706f3f3b1955791441ed5b5b0c5acf17f0f1b113de40f1d74cd11df59 b200-gke-cos-training-kubeflow: de66551f631b79451dd86ba10479748dcf7a911cc70ae538f3fdea5b0a8dd642 bcm-inference: d7fe93b0a299ae87b15cf7fbb81a78b72b4d868a25c6fcbb70aa26f97a31f548 gb200-any: e96732f58ce13d20e6b355afa20346bc25d88a9197d303064bdb5de91404279a -gb200-eks-ubuntu-inference-dynamo: 668a1862791c5d41923f66ef263603b63f56d384e95e1579049cb4fc793d96ee +gb200-eks-ubuntu-inference-dynamo: 4a6f96ddb52b2e3b787ee0f65b6d4fd11052150f98ee17de4b4afe4401727619 gb200-eks-ubuntu-training-kubeflow: b13b8d7ac8e45fedf00df2eb39699ddcdb5bd4c7bc94478c465aa582deb0b769 gb200-eks-ubuntu-training-slurm: 03cac6712a2749ca6bb75f472544659c6a95d408bbddc56ac94aef5dc03c335a -gb200-oke-ubuntu-inference-dynamo: d65471fc0ace996a5b46b20c884b071ca33080c4f98b22d3fcfa59ff9770e9fb +gb200-oke-ubuntu-inference-dynamo: 9b771c100146719cd0a5534809dc4f0d70c84c19c62024fcae0aff20d355fa39 gb200-oke-ubuntu-training-kubeflow: 905229e58cd9920e1b4ae57405518d482e3d098f1b01aa27509d9447cb6b2f55 gb300-any: 51e94d57f8bf60b2d2920c8255a9933e2f1fcb8c8a594793c7666173154b03fe -gb300-eks-ubuntu-inference-dynamo: 172200808700cc9747dd372f61b2cf32bd2b5ab8e7ddf2ebbcb23c81f22603ed +gb300-eks-ubuntu-inference-dynamo: b61772b86011b7a39ba13d00dda699979e888846a00e72235a3f4bc573bb96f0 gb300-eks-ubuntu-training-kubeflow: d187e3797e92b203c361518b3fbc5af76acd0a62071bcf3d3fa2c093cd94f027 -h100-aks-ubuntu-inference-dynamo: 7830de226b961b7d3c3165dc559c2669d429939b800a87a54b8427bc157dc854 +h100-aks-ubuntu-inference-dynamo: 9d4cb5e4cf170972b205aa483209d13ade1a94aea5b1f4c870f8c0146d3bfc2e h100-aks-ubuntu-training-kubeflow: 30caffd8764966e7432672b1dcd74861012beaaf5ae0aeb32cd67ebca0c102af h100-aks-ubuntu-training-slurm: b1cc76701977baab3aca432f0022f8c0842c547058cf3fbbad261db3b1c068b3 h100-any: 63ec1b86ddeee974b2da15b1f8eafed9efab968a472b09d6179e1d117e667bc2 h100-bcm-ubuntu-training: 657509ca168955fd6624efe6c7abab91040df4ac7b0aedf8a7ed4f5e7d00caa4 -h100-eks-ubuntu-inference-dynamo: 7d16e73b8ce23edbccfc8f3ddc2d5dc000c5e6813240f157f1f8230efabe8d4e +h100-eks-ubuntu-inference-dynamo: b2db1bb4bb62dfa0790f9b77cb34b16f6261970b1420214f37b7ffa32bcfd5b5 h100-eks-ubuntu-inference-nim: 0b3525d5cf7b471e27556d7b68f05e5b8c606d90a62cbf5d2b6dbeeb4aeff67d h100-eks-ubuntu-training-kubeflow: f031ab2be281a5401cd3426fd5b8cb17020944938f38ddeaeb63540c96c149f1 h100-eks-ubuntu-training-slurm: ff33f128fa254c9057c672a3c9b98452b7a648f20284ada0479446ca2b2b1f8f -h100-gke-cos-inference-dynamo: eebc1ae69635ed9e4732adf88ec81507c4ea23fd3875084d7943a6710b028e0c +h100-gke-cos-inference-dynamo: 141753509e44bba98e54d75eab96a0b720c987bd06ec8a4922ed42dfd4cc5006 h100-gke-cos-training-kubeflow: f762c480f098f1f936ce7ab749f372fcfebf5d4ee11d06dbd473449bbb530e9a h100-gke-cos-training-slurm: 28697b311b2d218e5bc88499720ae21f592bf751ec55b1f2975b9d7c37427b9f -h100-kind-inference-dynamo: 2a7160eb1d0b494fd0b8dd7a89496ac11969f3ab35969264fd83d588dc70def0 +h100-kind-inference-dynamo: 76360ba5eb316ce1814af34e812298e69004e1d82af09a9473ca9384f2126e72 h100-kind-training-kubeflow: e77c4afe346b00d66a7329b5d74dc655c0219f15514dcbf58007c8db27564de3 h100-kind-training-slurm: 76f993dd8d5e6dc113e38e98b01d99c1fa26157093e808345b09dfb5410a5059 h200-any: d852f8915d79b5a8e527116f2aeede59df1d2d68969206484f40184afa8f5ce9 @@ -47,8 +47,8 @@ monitoring-hpa: b86b773c621d5eca9fafc1e12750427111c0865a762496ce6aba28cf4ee4aa7a ocp-inference-nim: bb3c7fa78c972241b47abedbff3069d4fad35ab0c64d5f0172c633d8ed610b27 ocp-training: 1f457d2c2aed921c2a3281cca1a55b1181d979cc99f07c1b51b9ce827379b401 rtx-pro-6000-any: ad8b509fb7f6c25c765d3b9112b03594e3aac99a48a9e6c11df790b131488284 -rtx-pro-6000-eks-ubuntu-inference-dynamo: bedbdf1b531f3bb8de74e38ff4df094297a572698dffa121aa3ee91d5918c942 -rtx-pro-6000-eks-ubuntu-inference-nim: 59cfc17d0cfa2429e637cf3cecfa2e1d397c8d3d2dc7f11c6ec5995d392a5382 -rtx-pro-6000-eks-ubuntu-training-kubeflow: 6b1e4c1b3e6eaab010dd12ed72f7374e6e206ebb065a437ec8ca56518e282f32 -rtx-pro-6000-lke-ubuntu-inference: b207bf93e21bcad9c3d2e4b59dd6696cd12e8967e7a5e9c7ba6fce883e7943c0 -rtx-pro-6000-lke-ubuntu-training: 2710efe3081974b752f51a883a99e063071abfa1038a3a49e76cbac711b8ca00 +rtx-pro-6000-eks-ubuntu-inference-dynamo: 84cde160654f85a6e5fbe1f4f26f9d9c08efe388711678641040794f5b84ccb8 +rtx-pro-6000-eks-ubuntu-inference-nim: c99ee6060908f8edb35988b6a27e72f8189d55ee9365c48307335aee3557834e +rtx-pro-6000-eks-ubuntu-training-kubeflow: 07801d7883014186f8606926546b268c429c75245a4caa10cfdc6f006320e5da +rtx-pro-6000-lke-ubuntu-inference: e9d3717a2e093104fb3163a964150e4b44ec7f34a1958c76cc6e385c87e3f897 +rtx-pro-6000-lke-ubuntu-training: f09be1374361894e6cc7f60548b2a6003450405abb03c5a999e00132fbee2e69 diff --git a/pkg/recipe/testdata/catalog_parity_golden.yaml b/pkg/recipe/testdata/catalog_parity_golden.yaml index ccc316a70..2e8b4acd7 100644 --- a/pkg/recipe/testdata/catalog_parity_golden.yaml +++ b/pkg/recipe/testdata/catalog_parity_golden.yaml @@ -9,31 +9,31 @@ a100-eks-ubuntu-training-kubeflow: a743c24bf62271a407b6608dd6099113ce4baaabe9506 a100-gke-cos-training-kubeflow: a8f9afa0cf6045d7789d1385b5cdabe654170da00a3894de446c34100dd60acb a100-oke-ubuntu-training-kubeflow: 2bc755fc494c4db63788544b7bad2c98dfc0b4322a1cbe65364993fb40e38db6 b200-any: 74665d86b2781705d2ecc7e28a7c45ecac931560095bacdd48cfcc5249714ac0 -b200-gke-cos-inference-dynamo: 72835241a6465ce0edb61333a18f34a97a48ce1e180d8bdb695755ede74de105 +b200-gke-cos-inference-dynamo: de879f8526521e4c5198cee2c91f29b5ed04ec8ab64a9e28def821341105c93d b200-gke-cos-training-kubeflow: 5d8da216a34ed3935c2b0f0732f3bcf1187081585bce4f28610330a03e7b7ab6 bcm-inference: fbad7e9704b9fbcd099ce9ea0799cecb0a0eb0fbc46f8b758ea5c1299be71db4 gb200-any: 55cd7d2a2d33f677f10d58a1ddfbd02b79026e7a959d19d244c229437f8d8b69 -gb200-eks-ubuntu-inference-dynamo: 77cc897bd4b610cb0bf40f7b0ee1063c83b1af5065f5bbdad5ef026679135ce1 +gb200-eks-ubuntu-inference-dynamo: 8aeaa6b60e36f20e3321c14160d3c122f3583937ed277bfff499ca4acbb691a2 gb200-eks-ubuntu-training-kubeflow: 8510ee4d66330a42add297565d531c8daa5d525006b0afa1e07dd442c72c8f36 gb200-eks-ubuntu-training-slurm: ecafb0d201f905a1a09556394cec2bdc601de3c556e9c7502bc98d998d3fe17e -gb200-oke-ubuntu-inference-dynamo: d99cb6ca728d1ff0f4c56e4986d3b8361c280b5de526c3651614d663d7d91e14 +gb200-oke-ubuntu-inference-dynamo: 6ee3c58c8363a91d84334817fddef175e0bde63aee6f710b467a941953f6fc4d gb200-oke-ubuntu-training-kubeflow: 21b58a301871cffdd40de2c10fd4f6ec55833a9621a5eebf864816ed941d8adb gb300-any: e46a40fd6441b6e1348d8b9bcb60ef78623e22980f2af75b466b7aaa2b6a9960 -gb300-eks-ubuntu-inference-dynamo: b228697b4c1e5040a853b602b6c41dcb6bfd7a7ba002b7bddbe688388a809c21 +gb300-eks-ubuntu-inference-dynamo: 414ea9efccad9f9b1f7bfdb738829f0c9310ba3621bae6d9da18ae6cce8137ad gb300-eks-ubuntu-training-kubeflow: 9e04ac26502c5e3968a08257062ac01fd53d28962c408b72e68f6a95db22ea8f -h100-aks-ubuntu-inference-dynamo: 85ccdf260249a67a5a1ff965fd0226e053bd332c4ec5bbac22e1e2c006b2da0c +h100-aks-ubuntu-inference-dynamo: d941fc63190422765c02db2f5f017da7131adc687ca1e25dfa2b53b120117787 h100-aks-ubuntu-training-kubeflow: a8ad56b9c749ec8cc84d3c866ed8520cf27bc364a067938101ad635633071dc2 h100-aks-ubuntu-training-slurm: 9791558c8e0e44b8462511d5ef03fa21c26bf0a3993fe17d11c4d9e8c4420605 h100-any: 887124130cfc77b01a0c3dc8385782de13b4ad359314e607a6034443f803884c h100-bcm-ubuntu-training: 790117ef6e991a27720954877e6f9e0e6d54b6f32c3988a22c388c104663c8ae -h100-eks-ubuntu-inference-dynamo: 1a49679c9fc814596c2cdbcccba7baf53409d53bc21195340bfb4d21ab2c2d51 +h100-eks-ubuntu-inference-dynamo: 9ecd22288407079e2255adbe353c25fbf441792f827baf7066c3df5cd50eb68e h100-eks-ubuntu-inference-nim: 6db15a210537c5695abbf7ebb57c2cfc33c4cab97ce9b0007d366bdc1bda33e6 h100-eks-ubuntu-training-kubeflow: 677de790e9697c102e850c2394d521fdba4ba6853004f196c65f59abba051ea6 h100-eks-ubuntu-training-slurm: 85ad4d6705e0ea7bc73d91fb9a1e99e5026adbe9dc6a322ac0bb3187fbb1ac78 -h100-gke-cos-inference-dynamo: 476f02ba3f72a6b81d1adfb01ca4acb1b47982028e03304f41e12c9df1b0d446 +h100-gke-cos-inference-dynamo: 9f6f269901e88eb29b00b3bdcfeba8ee7e39d3b5ed654257b0d3e2334918cc2d h100-gke-cos-training-kubeflow: bc98f3822cca567c496a91441919089f0d260a2f8cb8fedf6c678a89bf03cfb7 h100-gke-cos-training-slurm: f1aad8397ac49789794875ca0836189be018c120c98ac718edf9f5fd90356b0c -h100-kind-inference-dynamo: 33959156bba4e30c4a1336e2f1d79e10eb0de1946847f70b0c1090201a610f6b +h100-kind-inference-dynamo: 1c4cf3a98057275dadd724b0f068f32c4e9a8ae8744569d3a2814e7336938344 h100-kind-training-kubeflow: b5dfa79c42619c5b5995f932af56f343e4d0588344e8d37d36cab7483d1cee03 h100-kind-training-slurm: 8549b91c859978283d7371666565273a1c75e2908d9167ccaafa75f019b4d6de h200-any: d8cce8f74339053cb6c3ce33d99e12012f73b8c675a77403b42d6e68d2a5d53d @@ -47,8 +47,8 @@ monitoring-hpa: 8206ceb11958dadac846292bb693e7757762528d99d00316e4aa427ec4cb3ea9 ocp-inference-nim: f1f99e6e99b4f7084ab4cba65e38dddbefb9d7bcf9579f16c86d5f76508b232f ocp-training: 4c5d4e277307f9f08c7ed5a47d79e61b7a8bbc5970ba6554a0a4471b4e7d405e rtx-pro-6000-any: c54af648defc635cb8540064f4eb1cf605e4a95dd3f221a089cdfcc29d409438 -rtx-pro-6000-eks-ubuntu-inference-dynamo: 6448f2b42eec2f52df07bee010ab433c96daf2f592034bb07dcf6519b365db12 -rtx-pro-6000-eks-ubuntu-inference-nim: a63f7ae7621c251b49f3aeec1b4f18add34e29206f2d33a804694a43e87c1fb9 -rtx-pro-6000-eks-ubuntu-training-kubeflow: 95a0fd3483bf06446c640add474f260d7f0691f2049579f251871b466a204e1e -rtx-pro-6000-lke-ubuntu-inference: fe72e435418de60273524fb8b502a85c727f90adc1603cde8456b8914bec0bd9 -rtx-pro-6000-lke-ubuntu-training: e48c0b2459d94cfa4e0cb487290762dc5ba76f46a95323d5421dab569358212e +rtx-pro-6000-eks-ubuntu-inference-dynamo: 9012a9b9108e1f76dc129bdc2407bef55ef1b815dd93e3a484dc9ff8dfd6b7ba +rtx-pro-6000-eks-ubuntu-inference-nim: b226e782c924bfabec2b20a9533c6eac575cb9ad310a7e9cc6e0d28202e15a49 +rtx-pro-6000-eks-ubuntu-training-kubeflow: b567ecb933095717694b982d0650a349dd5f9929eb6c3d39a1b0e1d4fba73772 +rtx-pro-6000-lke-ubuntu-inference: aeb331c840ec81628163a7056ffed7295ffd9d57b05fa3b8b0ac2e96f36baf51 +rtx-pro-6000-lke-ubuntu-training: 2ddcfbd4a02c0ca4cce94187fe3ce8b421cd5436e25a1d387a6666c475cb169a