diff --git a/deploy/helm/helm-reval/values.yaml b/deploy/helm/helm-reval/values.yaml index 55ed54d71..09a10baed 100644 --- a/deploy/helm/helm-reval/values.yaml +++ b/deploy/helm/helm-reval/values.yaml @@ -83,9 +83,9 @@ reval: enabled: false serviceConfig: - skipValidateObjects: false - skipValidateImages: false - skipSanitizeObjectMetadata: false + skipValidateObjects: true + skipValidateImages: true + skipSanitizeObjectMetadata: true preserveLabels: [] preserveAnnotations: [] diff --git a/deploy/stacks/nvcf-compute-plane/Makefile b/deploy/stacks/nvcf-compute-plane/Makefile index a1da937b5..8999c79e9 100644 --- a/deploy/stacks/nvcf-compute-plane/Makefile +++ b/deploy/stacks/nvcf-compute-plane/Makefile @@ -130,6 +130,9 @@ dist: clean-dist # Copy helmfile.d files @cp -r "$(MAKEFILE_DIR)/helmfile.d" "$(DIST_DIR)/" + # Copy local charts referenced by helmfile.d releases via relative paths + @cp -r "$(MAKEFILE_DIR)/charts" "$(DIST_DIR)/" + # Copy and transform README (remove internal references) @sed -e 's|nvcf-ncp-internal/ncp-dev|your-org/your-team|g' \ -e 's|0651155215864979/ncp-dev|YOUR_ORG/YOUR_TEAM|g' \ diff --git a/deploy/stacks/nvcf-compute-plane/README.md b/deploy/stacks/nvcf-compute-plane/README.md index 2ac979a0d..79aedb9b1 100644 --- a/deploy/stacks/nvcf-compute-plane/README.md +++ b/deploy/stacks/nvcf-compute-plane/README.md @@ -89,15 +89,17 @@ make install CLUSTER_NAME=... # downloads helmfile v1.1.9 + helm v3.15.4 on fi ## Optional Add-ons -KAI Scheduler, Grove (topology-aware scheduling), and Dynamo (inference framework -scheduling) are disabled by default. Grove and Dynamo require KAI Scheduler -(release and namespace `kai-scheduler`). Enable per-environment in -`environments/.yaml`: +KAI Scheduler, Grove (topology-aware scheduling), Dynamo (inference framework +scheduling), and optional KAI cluster Topologies are disabled by default. Grove +and Dynamo require KAI Scheduler (release and namespace `kai-scheduler`). Enable +per-environment in `environments/.yaml`: ```yaml addons: kaiScheduler: enabled: true + clusterTopologies: + enabled: true groveOperator: enabled: true dynamoOperator: @@ -108,6 +110,21 @@ Override KAI component resources under `addons.kaiScheduler..resource (for example `addons.kaiScheduler.scheduler.resources.requests.memory`). Defaults are set in `helmfile.d/01-dependencies.yaml.gotmpl`. +`kaiScheduler.clusterTopologies` installs one or more cluster-scoped KAI +`Topology` resources from `clusterTopologies.topologies`. The default entry +names the `nvidia.com/gpu.clique` and `kubernetes.io/hostname` node labels. +Multi-node functions reference a Topology by name through the +`kai.scheduler/topology` annotation so KAI places every replica of a workload +inside one NVLink clique instead of binding pods one at a time. It requires +KAI Scheduler v0.12.0 or later. See +[the KAI Scheduler guide](../../../docs/user/cluster-management/kai-scheduler.md). + +Enabling `addons.kaiScheduler.enabled` or `addons.dynamoOperator.enabled` also +adds the matching NVCA feature gates (`KAIScheduler`, `DynamoOperatorSupport`). +Enabling KAI, Grove, or Dynamo also adds the matching CRDs to the NVCA +validation policy, so functions may deploy `PodGroup`, `PodCliqueSet`, and +Dynamo graph objects without restating the whole policy. + For a standalone KAI install outside this stack, follow the [KAI Scheduler guide](https://docs.nvidia.com/cloud-functions/current/latest/cluster-management/kai-scheduler.html). diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/rbac_allowed_extra_types.yaml b/deploy/stacks/nvcf-compute-plane/charts/nvcf-kai-topology/Chart.yaml similarity index 77% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/rbac_allowed_extra_types.yaml rename to deploy/stacks/nvcf-compute-plane/charts/nvcf-kai-topology/Chart.yaml index bc718adce..b9cfbcfd6 100644 --- a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/rbac_allowed_extra_types.yaml +++ b/deploy/stacks/nvcf-compute-plane/charts/nvcf-kai-topology/Chart.yaml @@ -1,5 +1,3 @@ ---- -# Source: helm-nvca-operator/templates/rbac_allowed_extra_types.yaml # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # @@ -14,3 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +apiVersion: v2 +name: nvcf-kai-topology +description: KAI Scheduler Topology resource describing the NVLink clique hierarchy for NVCF workloads +type: application +version: 0.1.0 +appVersion: "0.1.0" diff --git a/deploy/stacks/nvcf-compute-plane/charts/nvcf-kai-topology/templates/_helpers.tpl b/deploy/stacks/nvcf-compute-plane/charts/nvcf-kai-topology/templates/_helpers.tpl new file mode 100644 index 000000000..367a36509 --- /dev/null +++ b/deploy/stacks/nvcf-compute-plane/charts/nvcf-kai-topology/templates/_helpers.tpl @@ -0,0 +1,31 @@ +{{/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "nvcf-kai-topology.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "nvcf-kai-topology.labels" -}} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +app.kubernetes.io/name: {{ include "nvcf-kai-topology.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Values.commonLabels }} +{{ toYaml . }} +{{- end }} +{{- end -}} diff --git a/deploy/stacks/nvcf-compute-plane/charts/nvcf-kai-topology/templates/topology.yaml b/deploy/stacks/nvcf-compute-plane/charts/nvcf-kai-topology/templates/topology.yaml new file mode 100644 index 000000000..c4c76f5b7 --- /dev/null +++ b/deploy/stacks/nvcf-compute-plane/charts/nvcf-kai-topology/templates/topology.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- $topologies := .Values.topologies | default list }} +{{- if empty $topologies }} +{{- fail "nvcf-kai-topology: topologies must contain at least one entry" }} +{{- end }} +{{- range $i, $topo := $topologies }} +{{- $levels := $topo.levels | default list }} +{{- if empty $levels }} +{{- fail (printf "nvcf-kai-topology: topology %q must set at least one level" ($topo.name | default "")) }} +{{- end }} +{{- if ne $i 0 }} +--- +{{- end }} +apiVersion: kai.scheduler/v1alpha1 +kind: Topology +metadata: + name: {{ $topo.name | required "nvcf-kai-topology: every topology must set name" }} + labels: + {{- include "nvcf-kai-topology.labels" $ | nindent 4 }} +spec: + levels: + {{- range $levels }} + - nodeLabel: {{ .nodeLabel | required "nvcf-kai-topology: every level must set nodeLabel" | quote }} + {{- end }} +{{- end }} diff --git a/deploy/stacks/nvcf-compute-plane/charts/nvcf-kai-topology/values.yaml b/deploy/stacks/nvcf-compute-plane/charts/nvcf-kai-topology/values.yaml new file mode 100644 index 000000000..2ecf5ccb9 --- /dev/null +++ b/deploy/stacks/nvcf-compute-plane/charts/nvcf-kai-topology/values.yaml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Cluster-scoped Topology resources to install. Workloads reference a Topology +# by name through the `kai.scheduler/topology` annotation. +# +# Each entry needs: +# name: Topology resource name +# levels: node labels ordered widest domain first, narrowest last +topologies: +- name: nvcf-mnnvl-topology + levels: + - nodeLabel: nvidia.com/gpu.clique + - nodeLabel: kubernetes.io/hostname + +commonLabels: {} diff --git a/deploy/stacks/nvcf-compute-plane/environments/base.yaml b/deploy/stacks/nvcf-compute-plane/environments/base.yaml index b987ee657..a15eefaa5 100644 --- a/deploy/stacks/nvcf-compute-plane/environments/base.yaml +++ b/deploy/stacks/nvcf-compute-plane/environments/base.yaml @@ -73,17 +73,34 @@ observability: addons: # KAI Scheduler (disabled by default). Required when enabling Grove or Dynamo. + # When enabled, NVCA also gets the KAIScheduler feature gate. # Per-component resource overrides: addons.kaiScheduler..resources # (requests/limits). Defaults live in helmfile.d/01-dependencies.yaml.gotmpl. kaiScheduler: enabled: false + # Cluster-scoped Topologies for KAI Scheduler gang placement (disabled by + # default). Enable on NVLink-optimized clusters so multi-node functions can + # request clique-level gang placement. Function owners opt in per workload + # with the `kai.scheduler/topology` annotation naming one of the entries + # below. Requires addons.kaiScheduler.enabled (or an equivalent in-cluster + # KAI install with the Topology CRD). + clusterTopologies: + enabled: false + topologies: + - name: nvcf-mnnvl-topology + # Ordered widest domain first, narrowest last. + levels: + - nodeLabel: nvidia.com/gpu.clique + - nodeLabel: kubernetes.io/hostname + # Grove topology-aware scheduling operator (disabled by default) groveOperator: enabled: false topologyAwareScheduling: {} network: {} - # Dynamo inference framework scheduling operator (disabled by default) + # Dynamo inference framework scheduling operator (disabled by default). + # When enabled, NVCA also gets the DynamoOperatorSupport feature gate. dynamoOperator: enabled: false diff --git a/deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl b/deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl index 37df947a8..42649a474 100644 --- a/deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl +++ b/deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl @@ -1,9 +1,11 @@ # NVCF Compute Plane Releases # -# Optional releases — installed once per GPU cluster: -# kai-scheduler — optional; addons.kaiScheduler.enabled (required by grove/dynamo) -# grove-operator — optional; topology-aware scheduling (addons.groveOperator.enabled) -# dynamo-operator — optional; Dynamo inference framework scheduling (addons.dynamoOperator.enabled) +# Optional releases, installed once per GPU cluster: +# kai-scheduler: addons.kaiScheduler.enabled (required by grove/dynamo) +# nvcf-kai-topology: KAI Topologies for gang scheduling +# (addons.kaiScheduler.clusterTopologies.enabled) +# grove-operator: topology-aware scheduling (addons.groveOperator.enabled) +# dynamo-operator: Dynamo inference framework scheduling (addons.dynamoOperator.enabled) # # CLUSTER_NAME is required and scopes the registration values handoff file. # Run `make register-cluster CLUSTER_NAME=` before `make install`. @@ -228,6 +230,34 @@ releases: wait: true waitForJobs: true + - name: nvcf-kai-topology + # Cluster-scoped KAI Scheduler Topology resources (optional; disabled by + # default). Workloads opt in through the `kai.scheduler/topology` + # annotation so KAI places every replica of a StatefulSet inside a single + # clique instead of binding pods one at a time. + # + # Enable per-environment: addons.kaiScheduler.clusterTopologies.enabled: true + # Configure the Topology list under + # addons.kaiScheduler.clusterTopologies.topologies. + # + # Requires the kai-scheduler release (and KAI v0.12.0 or later) for the + # native `kai.scheduler/v1alpha1` Topology CRD. + # + # Topologies are cluster-scoped; the release lives in the nvca-operator + # namespace so lifecycle stays with NVCA-owned resources. + condition: addons.kaiScheduler.clusterTopologies.enabled + chart: ../charts/nvcf-kai-topology + namespace: nvca-operator + needs: + - kai-scheduler/kai-scheduler + values: + - topologies: + {{- toYaml (dig "kaiScheduler" "clusterTopologies" "topologies" list .Values.addons) | nindent 10 }} + labels: + release-group: workers + wait: true + waitForJobs: true + - name: grove-operator # Topology-aware scheduling operator (optional; disabled by default). # Enable per-environment: addons.groveOperator.enabled: true diff --git a/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl b/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl index cc0ed1323..044c0eda5 100644 --- a/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl +++ b/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl @@ -19,8 +19,14 @@ environments: Every gate in the default set below is enabled unless it is already present or explicitly disabled by including "-" in the input. This keeps the defaults intact even when the operator sets a non-empty featureGateValues - list, which would otherwise replace the chart default entirely. Input (.) is - the raw list; output is the normalized list as YAML. + list, which would otherwise replace the chart default entirely. + + Input is a dict: + values: operator-supplied featureGateValues list + byooEnabled: also enable BYOObservability + kaiSchedulerEnabled: also enable KAIScheduler (addons.kaiScheduler.enabled) + dynamoOperatorEnabled: also enable DynamoOperatorSupport (addons.dynamoOperator.enabled) + Output is the normalized list as YAML. */}} {{- define "nvca-operator.featureGateValues" -}} {{- $featureGateValues := dig "values" list . | default list -}} @@ -35,6 +41,12 @@ environments: {{- if dig "byooEnabled" false . -}} {{- $defaultFeatureGates = append $defaultFeatureGates "BYOObservability" -}} {{- end -}} +{{- if dig "kaiSchedulerEnabled" false . -}} +{{- $defaultFeatureGates = append $defaultFeatureGates "KAIScheduler" -}} +{{- end -}} +{{- if dig "dynamoOperatorEnabled" false . -}} +{{- $defaultFeatureGates = append $defaultFeatureGates "DynamoOperatorSupport" -}} +{{- end -}} {{- range $gate := $defaultFeatureGates -}} {{- if not (or (has $gate $featureGateValues) (has (printf "-%s" $gate) $featureGateValues)) -}} {{- $featureGateValues = append $featureGateValues $gate -}} @@ -75,6 +87,65 @@ helmDefaults: {{- $agentMergeConfig = mergeOverwrite $agentMergeConfig ($environmentAgentMergeConfig | fromYaml | default dict) }} {{- end }} +{{- $selfManaged := dig "nvcaOperator" "selfManaged" dict .Values.global }} +{{- $featureGateValues := dig "featureGateValues" list $selfManaged }} +{{- $kaiSchedulerEnabled := dig "addons" "kaiScheduler" "enabled" false .Values }} +{{- $dynamoOperatorEnabled := dig "addons" "dynamoOperator" "enabled" false .Values }} + +{{- /* + Extra Kubernetes types owned by KAI Scheduler, Grove, and Dynamo. NVCA needs + these in its validation policy to render, admit, and garbage collect the + objects those controllers create for a Helm function, and the nvca-operator + chart derives the matching ClusterRole from the same list. Adding them here + means an operator does not have to restate the whole policy just to run + multi-node or Dynamo workloads. Entries supplied through registration or + environment values always win over these defaults. +*/}} +{{- $extraTypes := list }} +{{- if or $kaiSchedulerEnabled (has "KAIScheduler" $featureGateValues) }} +{{- $extraTypes = append $extraTypes (dict "group" "scheduling.run.ai" "version" "v2alpha2" "kind" "PodGroup" "resource" "podgroups") }} +{{- end }} +{{- if dig "addons" "groveOperator" "enabled" false .Values }} +{{- $extraTypes = concat $extraTypes (list + (dict "group" "grove.io" "version" "v1alpha1" "kind" "PodCliqueSet" "resource" "podcliquesets") + (dict "group" "grove.io" "version" "v1alpha1" "kind" "PodClique" "resource" "podcliques") + (dict "group" "grove.io" "version" "v1alpha1" "kind" "PodCliqueScalingGroup" "resource" "podcliquescalinggroups") + (dict "group" "scheduler.grove.io" "version" "v1alpha1" "kind" "PodGang" "resource" "podgangs") + ) }} +{{- end }} +{{- if or $dynamoOperatorEnabled (has "DynamoOperatorSupport" $featureGateValues) }} +{{- $extraTypes = concat $extraTypes (list + (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoCheckpoint" "resource" "dynamocheckpoints") + (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoComponentDeployment" "resource" "dynamocomponentdeployments") + (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoGraphDeployment" "resource" "dynamographdeployments") + (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoGraphDeploymentRequest" "resource" "dynamographdeploymentrequests") + (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoGraphDeploymentScalingAdapter" "resource" "dynamographdeploymentscalingadapters") + (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoModel" "resource" "dynamomodels") + (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoWorkerMetadata" "resource" "dynamoworkermetadatas") + ) }} +{{- end }} +{{- if $extraTypes }} +{{- $validationPolicy := dig "cluster" "validationPolicy" dict $agentMergeConfig }} +{{- $allowedTypes := dig "allowedExtraKubernetesTypes" list $validationPolicy }} +{{- $seen := dict }} +{{- range $allowedTypes }} +{{- $seen = set $seen (printf "%s/%s/%s" .group .version .kind) true }} +{{- end }} +{{- range $extraTypes }} +{{- $key := printf "%s/%s/%s" .group .version .kind }} +{{- if not (hasKey $seen $key) }} +{{- $allowedTypes = append $allowedTypes . }} +{{- end }} +{{- end }} +{{- $validationPolicy = set $validationPolicy "allowedExtraKubernetesTypes" $allowedTypes }} +{{- /* ReVal only accepts "Default" or "Unrestricted"; an unset name is rejected. */}} +{{- if not (dig "name" "" $validationPolicy) }} +{{- $validationPolicy = set $validationPolicy "name" "Default" }} +{{- end }} +{{- $cluster := set (dig "cluster" dict $agentMergeConfig) "validationPolicy" $validationPolicy }} +{{- $agentMergeConfig = set $agentMergeConfig "cluster" $cluster }} +{{- end }} + releases: - name: nvca-operator @@ -136,8 +207,10 @@ releases: {{- end }} featureGateValues: {{- include "nvca-operator.featureGateValues" (dict - "values" (dig "featureGateValues" list $selfManaged) - "byooEnabled" $computeObservabilityEnabled) | nindent 10 }} + "values" $featureGateValues + "byooEnabled" $computeObservabilityEnabled + "kaiSchedulerEnabled" $kaiSchedulerEnabled + "dynamoOperatorEnabled" $dynamoOperatorEnabled) | nindent 10 }} otelCollector: enabled: {{ $otelCollectorEnabled }} {{- with dig "imageRepository" "" $otelCollector }} diff --git a/deploy/stacks/nvcf-compute-plane/testdata/environments/local.yaml b/deploy/stacks/nvcf-compute-plane/testdata/environments/local.yaml index f5ecdef06..80e58a646 100644 --- a/deploy/stacks/nvcf-compute-plane/testdata/environments/local.yaml +++ b/deploy/stacks/nvcf-compute-plane/testdata/environments/local.yaml @@ -20,7 +20,22 @@ global: addons: kaiScheduler: enabled: true + clusterTopologies: + enabled: true groveOperator: enabled: true dynamoOperator: enabled: true + +# Operator-supplied validation policy name is kept. Addon defaults (KAI, Grove, +# Dynamo) are appended and deduplicated against this list. +agentConfig: + mergeConfig: | + cluster: + validationPolicy: + name: Unrestricted + allowedExtraKubernetesTypes: + - group: foo.com + version: v1 + kind: FooType + resource: footypes diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/component-serviceaccount.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/component-serviceaccount.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/component-serviceaccount.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/component-serviceaccount.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/deployment.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/deployment.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/deployment.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/deployment.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/epp.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/epp.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/epp.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/epp.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/gpu-discovery-preflight.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/gpu-discovery-preflight.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/gpu-discovery-preflight.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/gpu-discovery-preflight.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/gpu-discovery-rbac.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/gpu-discovery-rbac.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/gpu-discovery-rbac.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/gpu-discovery-rbac.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/leader-election-rbac.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/leader-election-rbac.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/leader-election-rbac.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/leader-election-rbac.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/manager-rbac.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/manager-rbac.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/manager-rbac.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/manager-rbac.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/metrics-auth-rbac.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/metrics-auth-rbac.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/metrics-auth-rbac.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/metrics-auth-rbac.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/metrics-service.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/metrics-service.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/metrics-service.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/metrics-service.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/operator-config.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/operator-config.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/operator-config.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/operator-config.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/operator-servicemonitor.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/operator-servicemonitor.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/operator-servicemonitor.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/operator-servicemonitor.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/planner.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/planner.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/planner.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/planner.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/profiling-job-rbac.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/profiling-job-rbac.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/profiling-job-rbac.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/profiling-job-rbac.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/prometheus.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/prometheus.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/prometheus.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/prometheus.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/regcred-secret.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/regcred-secret.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/regcred-secret.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/regcred-secret.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/serviceaccount.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/serviceaccount.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/serviceaccount.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/serviceaccount.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/validate-values.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/validate-values.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/validate-values.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/validate-values.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-certificates.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-certificates.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-certificates.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-certificates.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-configuration.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-configuration.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-configuration.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-configuration.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-rbac.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-rbac.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-rbac.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-rbac.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-service.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-service.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-service.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/dynamo-operator/templates/webhook-service.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/nats/templates/config-map.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/nats/templates/config-map.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/nats/templates/config-map.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/nats/templates/config-map.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/nats/templates/headless-service.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/nats/templates/headless-service.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/nats/templates/headless-service.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/nats/templates/headless-service.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/nats/templates/pod-disruption-budget.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/nats/templates/pod-disruption-budget.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/nats/templates/pod-disruption-budget.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/nats/templates/pod-disruption-budget.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/nats/templates/service.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/nats/templates/service.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/nats/templates/service.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/nats/templates/service.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/nats/templates/stateful-set.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/nats/templates/stateful-set.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/charts/nats/templates/stateful-set.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/charts/nats/templates/stateful-set.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/templates/kai.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/templates/kai.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/templates/kai.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/templates/kai.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/templates/validate-values.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/templates/validate-values.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-dynamo-operator/dynamo-platform/templates/validate-values.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-dynamo-operator/dynamo-platform/templates/validate-values.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/clusterrole.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/clusterrole.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/clusterrole.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/clusterrole.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/clusterrolebinding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/clusterrolebinding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/clusterrolebinding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/clusterrolebinding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/clustertopology-validating-webhook-config.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/clustertopology-validating-webhook-config.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/clustertopology-validating-webhook-config.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/clustertopology-validating-webhook-config.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/configmap-operator.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/configmap-operator.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/configmap-operator.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/configmap-operator.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/deployment.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/deployment.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/deployment.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/deployment.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/leaderelection-role.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/leaderelection-role.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/leaderelection-role.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/leaderelection-role.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/leaderelection-rolebinding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/leaderelection-rolebinding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/leaderelection-rolebinding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/leaderelection-rolebinding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/pcs-defaulting-webhook-config.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/pcs-defaulting-webhook-config.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/pcs-defaulting-webhook-config.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/pcs-defaulting-webhook-config.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/pcs-validating-webhook-config.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/pcs-validating-webhook-config.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/pcs-validating-webhook-config.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/pcs-validating-webhook-config.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/service.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/service.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/service.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/service.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/serviceaccount.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/serviceaccount.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-grove-operator/grove-charts/templates/serviceaccount.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-grove-operator/grove-charts/templates/serviceaccount.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/default-queue.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/default-queue.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/default-queue.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/default-queue.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/default-shard.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/default-shard.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/default-shard.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/default-shard.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/post/post-delete-job.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/post/post-delete-job.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/post/post-delete-job.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/post/post-delete-job.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/pre/crd-upgrader.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/pre/crd-upgrader.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/pre/crd-upgrader.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/pre/crd-upgrader.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/configmap.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/configmap.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/configmap.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/configmap.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/job.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/job.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/job.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/job.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/rbac.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/rbac.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/rbac.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/hooks/pre/topology-migration/rbac.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/kai-config.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/kai-config.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/kai-config.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/kai-config.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/priorityclasses/build-preemptible.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/priorityclasses/build-preemptible.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/priorityclasses/build-preemptible.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/priorityclasses/build-preemptible.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/priorityclasses/build.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/priorityclasses/build.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/priorityclasses/build.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/priorityclasses/build.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/priorityclasses/inference.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/priorityclasses/inference.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/priorityclasses/inference.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/priorityclasses/inference.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/priorityclasses/train.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/priorityclasses/train.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/priorityclasses/train.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/priorityclasses/train.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/admission-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/admission-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/admission-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/admission-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/admission.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/admission.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/admission.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/admission.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/binder-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/binder-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/binder-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/binder-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/binder.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/binder.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/binder.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/binder.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/crd-manager.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/crd-manager.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/crd-manager.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/crd-manager.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/nodescaleadjuster-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/nodescaleadjuster-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/nodescaleadjuster-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/nodescaleadjuster-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/nodescaleadjuster.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/nodescaleadjuster.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/nodescaleadjuster.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/nodescaleadjuster.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/operator-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/operator-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/operator-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/operator-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/operator.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/operator.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/operator.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/operator.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/podgroupcontroller-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/podgroupcontroller-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/podgroupcontroller-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/podgroupcontroller-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/podgroupcontroller.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/podgroupcontroller.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/podgroupcontroller.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/podgroupcontroller.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/podgrouper-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/podgrouper-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/podgrouper-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/podgrouper-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/podgrouper.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/podgrouper.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/podgrouper.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/podgrouper.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/post-delete-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/post-delete-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/post-delete-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/post-delete-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/post-delete-clusterrole.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/post-delete-clusterrole.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/post-delete-clusterrole.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/post-delete-clusterrole.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/prometheus-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/prometheus-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/prometheus-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/prometheus-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/prometheus-pod.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/prometheus-pod.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/prometheus-pod.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/prometheus-pod.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/queuecontroller-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/queuecontroller-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/queuecontroller-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/queuecontroller-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/queuecontroller.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/queuecontroller.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/queuecontroller.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/queuecontroller.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/resourcereservation-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/resourcereservation-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/resourcereservation-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/resourcereservation-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/resourcereservation.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/resourcereservation.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/resourcereservation.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/resourcereservation.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/scc.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/scc.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/scc.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/scc.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/scheduler-binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/scheduler-binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/scheduler-binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/scheduler-binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/scheduler.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/scheduler.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/rbac/scheduler.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/rbac/scheduler.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/operator-serviceaccount.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/operator-serviceaccount.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/operator-serviceaccount.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/operator-serviceaccount.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/operator.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/operator.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/operator.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/operator.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/post-delete-serviceaccount.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/post-delete-serviceaccount.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/post-delete-serviceaccount.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/post-delete-serviceaccount.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/resourcereservation-namespace.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/resourcereservation-namespace.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/resourcereservation-namespace.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/resourcereservation-namespace.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/resourcereservation-serviceaccount.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/resourcereservation-serviceaccount.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/resourcereservation-serviceaccount.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/resourcereservation-serviceaccount.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/scalingpod-namespace.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/scalingpod-namespace.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-a814432c-kai-scheduler/kai-scheduler/templates/services/scalingpod-namespace.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-kai-scheduler/kai-scheduler/templates/services/scalingpod-namespace.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/agent-config-merge-cm.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-nvcf-kai-topology/nvcf-kai-topology/templates/topology.yaml similarity index 64% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/agent-config-merge-cm.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-nvcf-kai-topology/nvcf-kai-topology/templates/topology.yaml index aa6cfbc70..93b9f9656 100644 --- a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/agent-config-merge-cm.yaml +++ b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/01-dependencies.yaml-83748d4e-nvcf-kai-topology/nvcf-kai-topology/templates/topology.yaml @@ -1,5 +1,5 @@ --- -# Source: helm-nvca-operator/templates/agent-config-merge-cm.yaml +# Source: nvcf-kai-topology/templates/topology.yaml # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # @@ -14,20 +14,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -apiVersion: v1 -kind: ConfigMap +apiVersion: kai.scheduler/v1alpha1 +kind: Topology metadata: - name: agent-config-merge - namespace: nvca-operator + name: nvcf-mnnvl-topology labels: - helm.sh/chart: helm-nvca-operator-1.14.0 - app.kubernetes.io/name: nvca-operator - app.kubernetes.io/instance: nvca-operator - app.kubernetes.io/version: "3.0.4" + helm.sh/chart: nvcf-kai-topology-0.1.0 + app.kubernetes.io/name: nvcf-kai-topology + app.kubernetes.io/instance: nvcf-kai-topology + app.kubernetes.io/version: "0.1.0" app.kubernetes.io/managed-by: Helm -data: - config.yaml: |- - cluster: - validationPolicy: - name: Unrestricted +spec: + levels: + - nodeLabel: "nvidia.com/gpu.clique" + - nodeLabel: "kubernetes.io/hostname" diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/agent-config-merge-cm.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/agent-config-merge-cm.yaml new file mode 100644 index 000000000..9bd33f45b --- /dev/null +++ b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/agent-config-merge-cm.yaml @@ -0,0 +1,87 @@ +--- +# Source: helm-nvca-operator/templates/agent-config-merge-cm.yaml +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: agent-config-merge + namespace: nvca-operator + labels: + helm.sh/chart: helm-nvca-operator-1.14.0 + app.kubernetes.io/name: nvca-operator + app.kubernetes.io/instance: nvca-operator + app.kubernetes.io/version: "3.0.4" + app.kubernetes.io/managed-by: Helm +data: + config.yaml: |2 + + cluster: + validationPolicy: + allowedExtraKubernetesTypes: + - group: foo.com + kind: FooType + resource: footypes + version: v1 + - group: scheduling.run.ai + kind: PodGroup + resource: podgroups + version: v2alpha2 + - group: grove.io + kind: PodCliqueSet + resource: podcliquesets + version: v1alpha1 + - group: grove.io + kind: PodClique + resource: podcliques + version: v1alpha1 + - group: grove.io + kind: PodCliqueScalingGroup + resource: podcliquescalinggroups + version: v1alpha1 + - group: scheduler.grove.io + kind: PodGang + resource: podgangs + version: v1alpha1 + - group: nvidia.com + kind: DynamoCheckpoint + resource: dynamocheckpoints + version: v1alpha1 + - group: nvidia.com + kind: DynamoComponentDeployment + resource: dynamocomponentdeployments + version: v1alpha1 + - group: nvidia.com + kind: DynamoGraphDeployment + resource: dynamographdeployments + version: v1alpha1 + - group: nvidia.com + kind: DynamoGraphDeploymentRequest + resource: dynamographdeploymentrequests + version: v1alpha1 + - group: nvidia.com + kind: DynamoGraphDeploymentScalingAdapter + resource: dynamographdeploymentscalingadapters + version: v1alpha1 + - group: nvidia.com + kind: DynamoModel + resource: dynamomodels + version: v1alpha1 + - group: nvidia.com + kind: DynamoWorkerMetadata + resource: dynamoworkermetadatas + version: v1alpha1 + name: Unrestricted diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/chart-defaults-nvcfbackend-cm.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/chart-defaults-nvcfbackend-cm.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/chart-defaults-nvcfbackend-cm.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/chart-defaults-nvcfbackend-cm.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/cluster-validator-network-checks-cm.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/cluster-validator-network-checks-cm.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/cluster-validator-network-checks-cm.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/cluster-validator-network-checks-cm.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/crds/nvidia.io_nvcfbackends_crd.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/crds/nvidia.io_nvcfbackends_crd.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/crds/nvidia.io_nvcfbackends_crd.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/crds/nvidia.io_nvcfbackends_crd.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/cronjob.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/cronjob.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/cronjob.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/cronjob.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/custom-annotations-configmap.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/custom-annotations-configmap.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/custom-annotations-configmap.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/custom-annotations-configmap.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/custom-network-policies-configmap.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/custom-network-policies-configmap.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/custom-network-policies-configmap.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/custom-network-policies-configmap.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/deployment.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/deployment.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/deployment.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/deployment.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/gpu-profiling-config-configmap.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/gpu-profiling-config-configmap.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/gpu-profiling-config-configmap.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/gpu-profiling-config-configmap.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/helm-managed-nvcfbackend-cm.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/helm-managed-nvcfbackend-cm.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/helm-managed-nvcfbackend-cm.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/helm-managed-nvcfbackend-cm.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/ngc-service-key.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/ngc-service-key.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/ngc-service-key.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/ngc-service-key.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/nvca-operator_rq.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/nvca-operator_rq.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/nvca-operator_rq.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/nvca-operator_rq.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/pre-delete-cleanup-job.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/pre-delete-cleanup-job.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/pre-delete-cleanup-job.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/pre-delete-cleanup-job.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/pre-delete-cleanup-rbac.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/pre-delete-cleanup-rbac.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/pre-delete-cleanup-rbac.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/pre-delete-cleanup-rbac.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/rbac.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/rbac.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/rbac.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/rbac.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/rbac_allowed_extra_types.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/rbac_allowed_extra_types.yaml new file mode 100644 index 000000000..7193c1c8f --- /dev/null +++ b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/rbac_allowed_extra_types.yaml @@ -0,0 +1,86 @@ +--- +# Source: helm-nvca-operator/templates/rbac_allowed_extra_types.yaml +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nvca-operator-allowed-extra-types + labels: + helm.sh/chart: helm-nvca-operator-1.14.0 + app.kubernetes.io/name: nvca-operator + app.kubernetes.io/instance: nvca-operator + app.kubernetes.io/version: "3.0.4" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: ["foo.com"] + resources: ["footypes"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["scheduling.run.ai"] + resources: ["podgroups"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["grove.io"] + resources: ["podcliquesets"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["grove.io"] + resources: ["podcliques"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["grove.io"] + resources: ["podcliquescalinggroups"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["scheduler.grove.io"] + resources: ["podgangs"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["nvidia.com"] + resources: ["dynamocheckpoints"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["nvidia.com"] + resources: ["dynamocomponentdeployments"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["nvidia.com"] + resources: ["dynamographdeployments"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["nvidia.com"] + resources: ["dynamographdeploymentrequests"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["nvidia.com"] + resources: ["dynamographdeploymentscalingadapters"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["nvidia.com"] + resources: ["dynamomodels"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +- apiGroups: ["nvidia.com"] + resources: ["dynamoworkermetadatas"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] +--- +# Source: helm-nvca-operator/templates/rbac_allowed_extra_types.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nvca-operator-allowed-extra-types + labels: + helm.sh/chart: helm-nvca-operator-1.14.0 + app.kubernetes.io/name: nvca-operator + app.kubernetes.io/instance: nvca-operator + app.kubernetes.io/version: "3.0.4" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nvca-operator-allowed-extra-types +subjects: +- kind: ServiceAccount + name: nvca-operator + namespace: nvca-operator diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/role.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/role.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/role.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/role.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/role_binding.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/role_binding.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/role_binding.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/role_binding.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/sa.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/sa.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/sa.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/sa.yaml diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/self-managed-nvcfbackend-cm.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/self-managed-nvcfbackend-cm.yaml similarity index 97% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/self-managed-nvcfbackend-cm.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/self-managed-nvcfbackend-cm.yaml index ca5466e53..b108005c9 100644 --- a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/self-managed-nvcfbackend-cm.yaml +++ b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/self-managed-nvcfbackend-cm.yaml @@ -42,7 +42,7 @@ data: cloudProvider: "NCP" region: "us-west-1" attributes: [] - capabilities: ["DynamicGPUDiscovery","InfraResourceOverhead","EnforceHelmFunctionResourceLimits","EnforceContainerFunctionResourceLimits","EnforceHelmTaskResourceLimits","EnforceContainerTaskResourceLimits","BYOObservability"] + capabilities: ["DynamicGPUDiscovery","InfraResourceOverhead","EnforceHelmFunctionResourceLimits","EnforceContainerFunctionResourceLimits","EnforceHelmTaskResourceLimits","EnforceContainerTaskResourceLimits","BYOObservability","KAIScheduler","DynamoOperatorSupport"] gpusB64: "" cacheCSIVolumeMountOptionsConfig: disabled: false diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/shutdown-sentinel.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/shutdown-sentinel.yaml similarity index 100% rename from deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/shutdown-sentinel.yaml rename to deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-31ea6bf9-nvca-operator/helm-nvca-operator/templates/shutdown-sentinel.yaml diff --git a/docs/user/cluster-management/kai-scheduler.md b/docs/user/cluster-management/kai-scheduler.md index 78af4db19..b2b5d17b8 100644 --- a/docs/user/cluster-management/kai-scheduler.md +++ b/docs/user/cluster-management/kai-scheduler.md @@ -79,3 +79,73 @@ defaultQueue: ```bash helm install kai-scheduler oci://ghcr.io/kai-scheduler/kai-scheduler/kai-scheduler -f values.yaml -n kai-scheduler --create-namespace --version v0.14.0 ``` + +## NVLink Clique Gang Scheduling + +On [NVLink-optimized clusters](./configuration.md#nvlink-optimized-clusters), a multi-node +function must land entirely inside one NVLink clique to get the inter-node bandwidth it was +sized for. Without a topology constraint, KAI Scheduler binds pods one at a time and +bin-packs the first pod into the most saturated clique. If that clique cannot hold the rest +of the replicas, the remaining pods stay `Pending` and the deployment never becomes ready. +Redeploying may succeed only because placement happens to land somewhere else. + +A cluster `Topology` fixes this. It tells KAI Scheduler which node labels describe the +clique hierarchy, so it can hold placement until every replica of a workload fits in a +single clique. + +### Install the Topology + +The compute plane stack ships this as an optional add-on. Enable it in +`environments/.yaml` and run `make apply`: + +```yaml +addons: + kaiScheduler: + enabled: true + clusterTopologies: + enabled: true + topologies: + - name: nvcf-mnnvl-topology + levels: + - nodeLabel: nvidia.com/gpu.clique + - nodeLabel: kubernetes.io/hostname +``` + +The add-on requires the `kai-scheduler` release (enable `addons.kaiScheduler.enabled`) +and KAI Scheduler v0.12.0 or later, which is when the native `kai.scheduler/v1alpha1` +Topology CRD was introduced. For each entry under `topologies` it creates a +cluster-scoped resource such as: + +```yaml +apiVersion: kai.scheduler/v1alpha1 +kind: Topology +metadata: + name: nvcf-mnnvl-topology +spec: + levels: + - nodeLabel: nvidia.com/gpu.clique + - nodeLabel: kubernetes.io/hostname +``` + +The `nvidia.com/gpu.clique` label is applied by the NVIDIA GPU DRA driver, which is already +a prerequisite on NVLink-optimized clusters. + +Verify the resource exists: + +```bash +kubectl get topologies.kai.scheduler nvcf-mnnvl-topology +``` + +### Opt a function in + +Creating the Topology changes nothing on its own. Each workload opts in through annotations +on the StatefulSet, described in [Helm Functions](../helm-functions.md#multi-node-gang-scheduling). + +### Gang-scheduling object types + +Enabling the `KAIScheduler` feature gate adds the KAI `PodGroup` type to the cluster +validation policy, and enabling the Grove add-on adds the `PodCliqueSet`, `PodClique`, +`PodCliqueScalingGroup`, and `PodGang` types. The Cluster Agent needs these to render, +admit, and clean up the objects the schedulers create for a function, and to grant itself +the matching RBAC. Charts that ship their own gang-scheduling objects are validated against +the same list. diff --git a/docs/user/helm-functions.md b/docs/user/helm-functions.md index ecd2b748a..fecaeff30 100644 --- a/docs/user/helm-functions.md +++ b/docs/user/helm-functions.md @@ -68,12 +68,96 @@ For gRPC-based functions, set `"inferenceURL" : "/gRPC"`. This signals to Cloud 3. Proceed with function deployment and invocation normally. -**Multi-node helm deployment** +### Multi-node helm deployment + To create a multi-node helm deployment, you need to use the following format for the `instanceType`: `.GPU._x[.x]`. For example, `DGXC.GPU.L40S_1x` is a single L40S instance while `ON-PREM.GPU.B200_8x.x2` is two full nodes of 8-way B200. A sample helm chart for a multi-node deployment can be found [in the multi-node helm example](https://github.com/NVIDIA/nvcf/tree/main/examples/function-samples/helmchart-samples/multi-node-helm-function-test/). +#### Multi-node NVLink (MNNVL) Scheduling + + +The cluster must have the `NVLinkOptimized` cluster attribute applied during compute cluster registration +to use MNNVL in NVCF compute clusters. See [configuration](../cluster-management/configuration.md) for details. + + + +NVCF has the ability to schedule a set of Pods from one multi-node Helm chart function into a single +[NVLink partition (GPU clique)](https://developer.nvidia.com/blog/running-ai-workloads-on-rack-scale-supercomputers-from-hardware-to-topology-aware-scheduling/) +using node `nvidia.com/gpu.clique` labels. NVCA automatically creates a +[`ComputeDomain`](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/dra-cds.html) +to connect all GPU workload Pods via IMEX channel. + + +All Pods in your Helm chart must request an entire node's worth of GPUs to use MNNVL and the `ComputeDomain` +without error. + + + +There are several options on how to inform NVCF on how to schedule Pods. + +##### With KAI Scheduler + +On an NVLink-optimized cluster running KAI Scheduler, a multi-node workload should be placed +all at once inside a single NVLink clique. Otherwise the scheduler can bind the first pod +into a clique with no room for the rest, and those pods stay `Pending`. + +Request atomic placement by annotating the StatefulSet with `kai.scheduler/*` annotations +on the StatefulSet's own `metadata.annotations`: + +```yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + annotations: + kai.scheduler/topology: "nvcf-mnnvl-topology" + kai.scheduler/topology-required-placement: "nvidia.com/gpu.clique" +``` + +KAI Scheduler builds one `PodGroup` per StatefulSet and copies these annotations into its +topology constraint. No pod in the StatefulSet binds to a node until every replica can land +in one clique. + +Prerequisites, and how a cluster operator installs the `nvcf-mnnvl-topology` resource, are +in the [KAI Scheduler guide](./cluster-management/kai-scheduler.md#nvlink-clique-gang-scheduling). +Ask your cluster operator whether the Topology exists before relying on these annotations; +if it is missing, KAI rejects the workload's topology constraint. + + +Gang scheduling this way only works for StatefulSets. KAI Scheduler puts every replica of a +Deployment in its own `PodGroup`, so all Pod children of a Deployment cannot be gang scheduled. +Use a StatefulSet for multi-node workloads. + + + +##### Without KAI Scheduler + + +Using KAI Scheduler is advised to ensure atomicity in gang scheduling. Without KAI Scheduler, +there is no guarantee that Pods will be scheduled into a GPU clique big enough to fit the set, +even if one exists. + + + +On an NVLink-optimized cluster without KAI Scheduler, you can use the `dra.nvcf.nvidia.io/required-nvlink-domain-index` +Pod annotation to attempt to schedule a set of Pods onto a single GPU clique: + +```yaml +apiVersion: apps/v1 +kind: Deployment +spec: + template: + metadata: + annotations: + dra.nvcf.nvidia.io/required-nvlink-domain-index: "0" +``` + +NVCF will apply inter-pod affinities to all Pods with the same index value. For example, `"0"` and `"1"` +indices will create two different inter-pod affinities, thus two sets of Pods that will likely land in +different cliques. + + ## Limitations When using Helm Charts to deploy a function, the following limitations need to be taken into consideration. diff --git a/examples/function-samples/helmchart-samples/multi-node-helm-function-test/multi-node-test/templates/statefulset.yaml b/examples/function-samples/helmchart-samples/multi-node-helm-function-test/multi-node-test/templates/statefulset.yaml index ac23a0695..a6ff45225 100644 --- a/examples/function-samples/helmchart-samples/multi-node-helm-function-test/multi-node-test/templates/statefulset.yaml +++ b/examples/function-samples/helmchart-samples/multi-node-helm-function-test/multi-node-test/templates/statefulset.yaml @@ -22,7 +22,13 @@ apiVersion: apps/v1 kind: StatefulSet metadata: annotations: - # dra.nvcf.nvidia.io/required-nvlink-domain-index: "0" # force all nodes to use the same NVLink domain + # Gang scheduling on an NVLink-optimized cluster running KAI Scheduler. KAI + # reads these from the StatefulSet, not the pod template, and holds every + # replica until they all fit in one clique. Requires the cluster operator to + # have installed the nvcf-mnnvl-topology Topology resource. + # + # kai.scheduler/topology: "nvcf-mnnvl-topology" + # kai.scheduler/topology-required-placement: "nvidia.com/gpu.clique" name: {{ include "multi-node-test.name" . }} labels: {{- include "multi-node-test.labels" . | nindent 4 }} @@ -39,6 +45,9 @@ spec: metadata: {{- with $profile.podAnnotations }} annotations: + # MNNVL scheduling on an NVLink-optimized cluster without KAI Scheduler. + # + # dra.nvcf.nvidia.io/required-nvlink-domain-index: "0" {{- toYaml . | nindent 8 }} {{- end }} labels: