[Draft] feat(nvcf-compute-plane): gang scheduling with KAI MNNVL topology, KAI/Grove primitives

deploy/helm/helm-reval/values.yaml

@@ -83,9 +83,9 @@ reval:
     enabled: false
 
   serviceConfig:
-    skipValidateObjects: false
-    skipValidateImages: false
-    skipSanitizeObjectMetadata: false
+    skipValidateObjects: true
+    skipValidateImages: true
+    skipSanitizeObjectMetadata: true
     preserveLabels: []
     preserveAnnotations: []
 

deploy/stacks/nvcf-compute-plane/Makefile

@@ -130,6 +130,9 @@ dist: clean-dist
 	# Copy helmfile.d files
 	@cp -r "$(MAKEFILE_DIR)/helmfile.d" "$(DIST_DIR)/"
 
+	# Copy local charts referenced by helmfile.d releases via relative paths
+	@cp -r "$(MAKEFILE_DIR)/charts" "$(DIST_DIR)/"
+
 	# Copy and transform README (remove internal references)
 	@sed -e 's|nvcf-ncp-internal/ncp-dev|your-org/your-team|g' \
 	     -e 's|0651155215864979/ncp-dev|YOUR_ORG/YOUR_TEAM|g' \

deploy/stacks/nvcf-compute-plane/README.md

@@ -89,15 +89,17 @@ make install CLUSTER_NAME=...   # downloads helmfile v1.1.9 + helm v3.15.4 on fi
 
 ## Optional Add-ons
 
-KAI Scheduler, Grove (topology-aware scheduling), and Dynamo (inference framework
-scheduling) are disabled by default. Grove and Dynamo require KAI Scheduler
-(release and namespace `kai-scheduler`). Enable per-environment in
-`environments/<env>.yaml`:
+KAI Scheduler, Grove (topology-aware scheduling), Dynamo (inference framework
+scheduling), and optional KAI cluster Topologies are disabled by default. Grove
+and Dynamo require KAI Scheduler (release and namespace `kai-scheduler`). Enable
+per-environment in `environments/<env>.yaml`:
 
 ```yaml
 addons:
   kaiScheduler:
     enabled: true
+    clusterTopologies:
+      enabled: true
   groveOperator:
     enabled: true
   dynamoOperator:
@@ -108,6 +110,21 @@ Override KAI component resources under `addons.kaiScheduler.<component>.resource
 (for example `addons.kaiScheduler.scheduler.resources.requests.memory`). Defaults
 are set in `helmfile.d/01-dependencies.yaml.gotmpl`.
 
+`kaiScheduler.clusterTopologies` installs one or more cluster-scoped KAI
+`Topology` resources from `clusterTopologies.topologies`. The default entry
+names the `nvidia.com/gpu.clique` and `kubernetes.io/hostname` node labels.
+Multi-node functions reference a Topology by name through the
+`kai.scheduler/topology` annotation so KAI places every replica of a workload
+inside one NVLink clique instead of binding pods one at a time. It requires
+KAI Scheduler v0.12.0 or later. See
+[the KAI Scheduler guide](../../../docs/user/cluster-management/kai-scheduler.md).
+
+Enabling `addons.kaiScheduler.enabled` or `addons.dynamoOperator.enabled` also
+adds the matching NVCA feature gates (`KAIScheduler`, `DynamoOperatorSupport`).
+Enabling KAI, Grove, or Dynamo also adds the matching CRDs to the NVCA
+validation policy, so functions may deploy `PodGroup`, `PodCliqueSet`, and
+Dynamo graph objects without restating the whole policy.
+
 For a standalone KAI install outside this stack, follow the
 [KAI Scheduler guide](https://docs.nvidia.com/cloud-functions/current/latest/cluster-management/kai-scheduler.html).
 

deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-8e61e521-nvca-operator/helm-nvca-operator/templates/rbac_allowed_extra_types.yaml → deploy/stacks/nvcf-compute-plane/charts/nvcf-kai-topology/Chart.yaml

@@ -1,5 +1,3 @@
----
-# Source: helm-nvca-operator/templates/rbac_allowed_extra_types.yaml
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -14,3 +12,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+apiVersion: v2
+name: nvcf-kai-topology
+description: KAI Scheduler Topology resource describing the NVLink clique hierarchy for NVCF workloads
+type: application
+version: 0.1.0
+appVersion: "0.1.0"

deploy/stacks/nvcf-compute-plane/charts/nvcf-kai-topology/templates/_helpers.tpl

@@ -0,0 +1,31 @@
+{{/*
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "nvcf-kai-topology.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "nvcf-kai-topology.labels" -}}
+helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+app.kubernetes.io/name: {{ include "nvcf-kai-topology.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- with .Values.commonLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}

deploy/stacks/nvcf-compute-plane/charts/nvcf-kai-topology/templates/topology.yaml

@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- $topologies := .Values.topologies | default list }}
+{{- if empty $topologies }}
+{{- fail "nvcf-kai-topology: topologies must contain at least one entry" }}
+{{- end }}
+{{- range $i, $topo := $topologies }}
+{{- $levels := $topo.levels | default list }}
+{{- if empty $levels }}
+{{- fail (printf "nvcf-kai-topology: topology %q must set at least one level" ($topo.name | default "<unnamed>")) }}
+{{- end }}
+{{- if ne $i 0 }}
+---
+{{- end }}
+apiVersion: kai.scheduler/v1alpha1
+kind: Topology
+metadata:
+  name: {{ $topo.name | required "nvcf-kai-topology: every topology must set name" }}
+  labels:
+    {{- include "nvcf-kai-topology.labels" $ | nindent 4 }}
+spec:
+  levels:
+  {{- range $levels }}
+  - nodeLabel: {{ .nodeLabel | required "nvcf-kai-topology: every level must set nodeLabel" | quote }}
+  {{- end }}
+{{- end }}

deploy/stacks/nvcf-compute-plane/charts/nvcf-kai-topology/values.yaml

@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Cluster-scoped Topology resources to install. Workloads reference a Topology
+# by name through the `kai.scheduler/topology` annotation.
+#
+# Each entry needs:
+#   name:   Topology resource name
+#   levels: node labels ordered widest domain first, narrowest last
+topologies:
+- name: nvcf-mnnvl-topology
+  levels:
+  - nodeLabel: nvidia.com/gpu.clique
+  - nodeLabel: kubernetes.io/hostname
+
+commonLabels: {}

deploy/stacks/nvcf-compute-plane/environments/base.yaml

@@ -73,17 +73,34 @@ observability:
 addons:
 
   # KAI Scheduler (disabled by default). Required when enabling Grove or Dynamo.
+  # When enabled, NVCA also gets the KAIScheduler feature gate.
   # Per-component resource overrides: addons.kaiScheduler.<component>.resources
   # (requests/limits). Defaults live in helmfile.d/01-dependencies.yaml.gotmpl.
   kaiScheduler:
     enabled: false
 
+    # Cluster-scoped Topologies for KAI Scheduler gang placement (disabled by
+    # default). Enable on NVLink-optimized clusters so multi-node functions can
+    # request clique-level gang placement. Function owners opt in per workload
+    # with the `kai.scheduler/topology` annotation naming one of the entries
+    # below. Requires addons.kaiScheduler.enabled (or an equivalent in-cluster
+    # KAI install with the Topology CRD).
+    clusterTopologies:
+      enabled: false
+      topologies:
+        - name: nvcf-mnnvl-topology
+          # Ordered widest domain first, narrowest last.
+          levels:
+            - nodeLabel: nvidia.com/gpu.clique
+            - nodeLabel: kubernetes.io/hostname
+
   # Grove topology-aware scheduling operator (disabled by default)
   groveOperator:
     enabled: false
     topologyAwareScheduling: {}
     network: {}
 
-  # Dynamo inference framework scheduling operator (disabled by default)
+  # Dynamo inference framework scheduling operator (disabled by default).
+  # When enabled, NVCA also gets the DynamoOperatorSupport feature gate.
   dynamoOperator:
     enabled: false

deploy/stacks/nvcf-compute-plane/helmfile.d/01-dependencies.yaml.gotmpl

@@ -1,9 +1,11 @@
 # NVCF Compute Plane Releases
 #
-# Optional releases — installed once per GPU cluster:
-#   kai-scheduler    — optional; addons.kaiScheduler.enabled (required by grove/dynamo)
-#   grove-operator   — optional; topology-aware scheduling (addons.groveOperator.enabled)
-#   dynamo-operator  — optional; Dynamo inference framework scheduling (addons.dynamoOperator.enabled)
+# Optional releases, installed once per GPU cluster:
+#   kai-scheduler:        addons.kaiScheduler.enabled (required by grove/dynamo)
+#   nvcf-kai-topology:    KAI Topologies for gang scheduling
+#                         (addons.kaiScheduler.clusterTopologies.enabled)
+#   grove-operator:       topology-aware scheduling (addons.groveOperator.enabled)
+#   dynamo-operator:      Dynamo inference framework scheduling (addons.dynamoOperator.enabled)
 #
 # CLUSTER_NAME is required and scopes the registration values handoff file.
 # Run `make register-cluster CLUSTER_NAME=<name>` before `make install`.
@@ -228,6 +230,34 @@ releases:
     wait: true
     waitForJobs: true
 
+  - name: nvcf-kai-topology
+    # Cluster-scoped KAI Scheduler Topology resources (optional; disabled by
+    # default). Workloads opt in through the `kai.scheduler/topology`
+    # annotation so KAI places every replica of a StatefulSet inside a single
+    # clique instead of binding pods one at a time.
+    #
+    # Enable per-environment: addons.kaiScheduler.clusterTopologies.enabled: true
+    # Configure the Topology list under
+    # addons.kaiScheduler.clusterTopologies.topologies.
+    #
+    # Requires the kai-scheduler release (and KAI v0.12.0 or later) for the
+    # native `kai.scheduler/v1alpha1` Topology CRD.
+    #
+    # Topologies are cluster-scoped; the release lives in the nvca-operator
+    # namespace so lifecycle stays with NVCA-owned resources.
+    condition: addons.kaiScheduler.clusterTopologies.enabled
+    chart: ../charts/nvcf-kai-topology
+    namespace: nvca-operator
+    needs:
+      - kai-scheduler/kai-scheduler
+    values:
+      - topologies:
+        {{- toYaml (dig "kaiScheduler" "clusterTopologies" "topologies" list .Values.addons) | nindent 10 }}
+    labels:
+      release-group: workers
+    wait: true
+    waitForJobs: true
+
   - name: grove-operator
     # Topology-aware scheduling operator (optional; disabled by default).
     # Enable per-environment: addons.groveOperator.enabled: true

deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl

@@ -19,8 +19,14 @@ environments:
   Every gate in the default set below is enabled unless it is already present or
   explicitly disabled by including "-<gate>" in the input. This keeps the
   defaults intact even when the operator sets a non-empty featureGateValues
-  list, which would otherwise replace the chart default entirely. Input (.) is
-  the raw list; output is the normalized list as YAML.
+  list, which would otherwise replace the chart default entirely.
+
+  Input is a dict:
+    values:                 operator-supplied featureGateValues list
+    byooEnabled:            also enable BYOObservability
+    kaiSchedulerEnabled:    also enable KAIScheduler (addons.kaiScheduler.enabled)
+    dynamoOperatorEnabled:  also enable DynamoOperatorSupport (addons.dynamoOperator.enabled)
+  Output is the normalized list as YAML.
 */}}
 {{- define "nvca-operator.featureGateValues" -}}
 {{- $featureGateValues := dig "values" list . | default list -}}
@@ -35,6 +41,12 @@ environments:
 {{- if dig "byooEnabled" false . -}}
 {{- $defaultFeatureGates = append $defaultFeatureGates "BYOObservability" -}}
 {{- end -}}
+{{- if dig "kaiSchedulerEnabled" false . -}}
+{{- $defaultFeatureGates = append $defaultFeatureGates "KAIScheduler" -}}
+{{- end -}}
+{{- if dig "dynamoOperatorEnabled" false . -}}
+{{- $defaultFeatureGates = append $defaultFeatureGates "DynamoOperatorSupport" -}}
+{{- end -}}
 {{- range $gate := $defaultFeatureGates -}}
 {{-   if not (or (has $gate $featureGateValues) (has (printf "-%s" $gate) $featureGateValues)) -}}
 {{-     $featureGateValues = append $featureGateValues $gate -}}
@@ -75,6 +87,65 @@ helmDefaults:
 {{- $agentMergeConfig = mergeOverwrite $agentMergeConfig ($environmentAgentMergeConfig | fromYaml | default dict) }}
 {{- end }}
 
+{{- $selfManaged := dig "nvcaOperator" "selfManaged" dict .Values.global }}
+{{- $featureGateValues := dig "featureGateValues" list $selfManaged }}
+{{- $kaiSchedulerEnabled := dig "addons" "kaiScheduler" "enabled" false .Values }}
+{{- $dynamoOperatorEnabled := dig "addons" "dynamoOperator" "enabled" false .Values }}
+
+{{- /*
+  Extra Kubernetes types owned by KAI Scheduler, Grove, and Dynamo. NVCA needs
+  these in its validation policy to render, admit, and garbage collect the
+  objects those controllers create for a Helm function, and the nvca-operator
+  chart derives the matching ClusterRole from the same list. Adding them here
+  means an operator does not have to restate the whole policy just to run
+  multi-node or Dynamo workloads. Entries supplied through registration or
+  environment values always win over these defaults.
+*/}}
+{{- $extraTypes := list }}
+{{- if or $kaiSchedulerEnabled (has "KAIScheduler" $featureGateValues) }}
+{{-   $extraTypes = append $extraTypes (dict "group" "scheduling.run.ai" "version" "v2alpha2" "kind" "PodGroup" "resource" "podgroups") }}
+{{- end }}
+{{- if dig "addons" "groveOperator" "enabled" false .Values }}
+{{-   $extraTypes = concat $extraTypes (list
+        (dict "group" "grove.io" "version" "v1alpha1" "kind" "PodCliqueSet" "resource" "podcliquesets")
+        (dict "group" "grove.io" "version" "v1alpha1" "kind" "PodClique" "resource" "podcliques")
+        (dict "group" "grove.io" "version" "v1alpha1" "kind" "PodCliqueScalingGroup" "resource" "podcliquescalinggroups")
+        (dict "group" "scheduler.grove.io" "version" "v1alpha1" "kind" "PodGang" "resource" "podgangs")
+      ) }}
+{{- end }}
+{{- if or $dynamoOperatorEnabled (has "DynamoOperatorSupport" $featureGateValues) }}
+{{-   $extraTypes = concat $extraTypes (list
+        (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoCheckpoint" "resource" "dynamocheckpoints")
+        (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoComponentDeployment" "resource" "dynamocomponentdeployments")
+        (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoGraphDeployment" "resource" "dynamographdeployments")
+        (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoGraphDeploymentRequest" "resource" "dynamographdeploymentrequests")
+        (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoGraphDeploymentScalingAdapter" "resource" "dynamographdeploymentscalingadapters")
+        (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoModel" "resource" "dynamomodels")
+        (dict "group" "nvidia.com" "version" "v1alpha1" "kind" "DynamoWorkerMetadata" "resource" "dynamoworkermetadatas")
+      ) }}
+{{- end }}
+{{- if $extraTypes }}
+{{-   $validationPolicy := dig "cluster" "validationPolicy" dict $agentMergeConfig }}
+{{-   $allowedTypes := dig "allowedExtraKubernetesTypes" list $validationPolicy }}
+{{-   $seen := dict }}
+{{-   range $allowedTypes }}
+{{-     $seen = set $seen (printf "%s/%s/%s" .group .version .kind) true }}
+{{-   end }}
+{{-   range $extraTypes }}
+{{-     $key := printf "%s/%s/%s" .group .version .kind }}
+{{-     if not (hasKey $seen $key) }}
+{{-       $allowedTypes = append $allowedTypes . }}
+{{-     end }}
+{{-   end }}
+{{-   $validationPolicy = set $validationPolicy "allowedExtraKubernetesTypes" $allowedTypes }}
+{{- /* ReVal only accepts "Default" or "Unrestricted"; an unset name is rejected. */}}
+{{-   if not (dig "name" "" $validationPolicy) }}
+{{-     $validationPolicy = set $validationPolicy "name" "Default" }}
+{{-   end }}
+{{-   $cluster := set (dig "cluster" dict $agentMergeConfig) "validationPolicy" $validationPolicy }}
+{{-   $agentMergeConfig = set $agentMergeConfig "cluster" $cluster }}
+{{- end }}
+
 releases:
 
   - name: nvca-operator
@@ -136,8 +207,10 @@ releases:
           {{- end }}
           featureGateValues:
           {{- include "nvca-operator.featureGateValues" (dict
-                "values" (dig "featureGateValues" list $selfManaged)
-                "byooEnabled" $computeObservabilityEnabled) | nindent 10 }}
+                "values" $featureGateValues
+                "byooEnabled" $computeObservabilityEnabled
+                "kaiSchedulerEnabled" $kaiSchedulerEnabled
+                "dynamoOperatorEnabled" $dynamoOperatorEnabled) | nindent 10 }}
           otelCollector:
             enabled: {{ $otelCollectorEnabled }}
             {{- with dig "imageRepository" "" $otelCollector }}