From ed21a67d0a782af0ffc83e912362cd70fa460156 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Sun, 5 Jul 2026 00:04:29 +0300 Subject: [PATCH] feat(vgpu-manager): default Fabric Manager to vGPU multitenancy mode SR-IOV vGPU on NVSwitch (HGX) systems requires NVIDIA Fabric Manager to run in vGPU multitenancy mode (FABRIC_MODE=2) rather than the packaged bare-metal default (FABRIC_MODE=0). Without it a whole-card vGPU guest cannot initialize CUDA (fails at cuInit). Default the vGPU Manager container to FABRIC_MODE=2 so its entrypoint can configure fabricmanager.cfg for NVSwitch hosts. Non-NVSwitch vGPU hosts do not start Fabric Manager, so the value is ignored there. A user-provided FABRIC_MODE in VGPUManager.Env takes precedence. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin --- controllers/object_controls.go | 21 ++++++++++ controllers/transforms_test.go | 75 ++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/controllers/object_controls.go b/controllers/object_controls.go index aa6a54930..6d3e91518 100644 --- a/controllers/object_controls.go +++ b/controllers/object_controls.go @@ -173,6 +173,15 @@ const ( OpenKernelModulesEnabledEnvName = "OPEN_KERNEL_MODULES_ENABLED" // KernelModuleTypeEnvName is the name of the driver-container envvar to set the desired kernel module type KernelModuleTypeEnvName = "KERNEL_MODULE_TYPE" + // FabricModeEnvName is the name of the vGPU-manager-container envvar that + // selects the NVIDIA Fabric Manager mode (the FABRIC_MODE key in + // fabricmanager.cfg) on NVSwitch systems. + FabricModeEnvName = "FABRIC_MODE" + // FabricModeVGPUMultitenancy is the FABRIC_MODE value that starts Fabric + // Manager in vGPU multitenancy mode, required for SR-IOV vGPU on NVSwitch + // (HGX) systems. FABRIC_MODE=0 (bare-metal/passthrough) is the packaged + // default; FABRIC_MODE=1 is shared-NVSwitch multitenancy. + FabricModeVGPUMultitenancy = "2" // MPSRootEnvName is the name of the envvar for configuring the MPS root MPSRootEnvName = "MPS_ROOT" // DefaultMPSRoot is the default MPS root path on the host @@ -3896,6 +3905,18 @@ func transformVGPUManagerContainer(obj *appsv1.DaemonSet, config *gpuv1.ClusterP } } + // On NVSwitch (HGX) systems, SR-IOV vGPU requires NVIDIA Fabric Manager to + // run in vGPU multitenancy mode (FABRIC_MODE=2) rather than the packaged + // bare-metal default (FABRIC_MODE=0); otherwise a whole-card vGPU guest + // cannot initialize CUDA. Default the vGPU Manager container to + // FABRIC_MODE=2 so its entrypoint can configure fabricmanager.cfg + // accordingly on NVSwitch hosts. Non-NVSwitch vGPU hosts do not start Fabric + // Manager, so the value is ignored there. A user-provided FABRIC_MODE in + // VGPUManager.Env is applied above and takes precedence. + if getContainerEnv(container, FabricModeEnvName) == "" { + setContainerEnv(container, FabricModeEnvName, FabricModeVGPUMultitenancy) + } + // mount any custom kernel module configuration parameters at /drivers if config.VGPUManager.KernelModuleConfig != nil && config.VGPUManager.KernelModuleConfig.Name != "" { volumeMounts, itemsToInclude, err := createConfigMapVolumeMounts(n, config.VGPUManager.KernelModuleConfig.Name, driversDir) diff --git a/controllers/transforms_test.go b/controllers/transforms_test.go index 11b7e98b7..a680c5e9f 100644 --- a/controllers/transforms_test.go +++ b/controllers/transforms_test.go @@ -4233,6 +4233,81 @@ func TestTransformVGPUManager(t *testing.T) { } } +// TestTransformVGPUManagerFabricMode verifies that the vGPU Manager container is +// defaulted to NVIDIA Fabric Manager vGPU multitenancy mode (FABRIC_MODE=2), +// which SR-IOV vGPU on NVSwitch (HGX) systems requires, and that a +// user-provided FABRIC_MODE in VGPUManager.Env is preserved. +func TestTransformVGPUManagerFabricMode(t *testing.T) { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Labels: map[string]string{ + nfdOSReleaseIDLabelKey: "ubuntu", + nfdOSVersionIDLabelKey: "24.04", + nfdKernelLabelKey: "6.8.0-60-generic", + commonGPULabelKey: "true", + }, + }, + } + mockClient := fake.NewFakeClient(node) + + baseSpec := func() *gpuv1.ClusterPolicySpec { + return &gpuv1.ClusterPolicySpec{ + VGPUManager: gpuv1.VGPUManagerSpec{ + Repository: "nvcr.io/nvidia", + Image: "vgpu-manager", + Version: "550.90.07", + ImagePullPolicy: "IfNotPresent", + DriverManager: gpuv1.DriverManagerSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "k8s-driver-manager", + ImagePullPolicy: "IfNotPresent", + Version: "v0.8.0", + }, + }, + } + } + + testCases := []struct { + description string + env []gpuv1.EnvVar + expectedFabricMode string + }{ + { + description: "defaults to vGPU multitenancy mode", + env: nil, + expectedFabricMode: "2", + }, + { + description: "user-provided FABRIC_MODE is preserved", + env: []gpuv1.EnvVar{{Name: "FABRIC_MODE", Value: "0"}}, + expectedFabricMode: "0", + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + ds := NewDaemonset(). + WithInitContainer(corev1.Container{Name: "k8s-driver-manager"}). + WithContainer(corev1.Container{Name: "nvidia-vgpu-manager-ctr"}) + cpSpec := baseSpec() + cpSpec.VGPUManager.Env = tc.env + + clusterPolicyCtrl := ClusterPolicyController{ + logger: ctrl.Log.WithName("test"), + client: mockClient, + operatorNamespace: "test-ns", + } + err := TransformVGPUManager(ds.DaemonSet, cpSpec, clusterPolicyCtrl) + require.NoError(t, err) + + container := findContainerByName(ds.Spec.Template.Spec.Containers, "nvidia-vgpu-manager-ctr") + require.NotNil(t, container) + require.Equal(t, tc.expectedFabricMode, getContainerEnv(container, "FABRIC_MODE")) + }) + } +} + func TestTransformDriverWithAdditionalConfig(t *testing.T) { node := &corev1.Node{ ObjectMeta: metav1.ObjectMeta{