feat: CDI-based NVIDIA GPU passthrough into gVisor actor containers

README.md

@@ -202,7 +202,7 @@ We provide several sample applications demonstrating Agent Substrate's capabilit
 4. **[Secret Agent](demos/agent-secret/README.md)**: Highlights Substrate's "Zero-Idle" self-suspension and re-animation of volatile process memory.
 
 ### Documentation & Guides
-* [API Configuration Guide](docs/api-guide.md): Detailed reference for configuring WorkerPools, ActorTemplates, Secrets, and Volumes.
+* [API Configuration Guide](docs/api-guide.md): Detailed reference for configuring WorkerPools, ActorTemplates, Secrets, Volumes, and [GPU worker pools](docs/api-guide.md#gpu-worker-pools).
 * [Full CLI Documentation](cmd/kubectl-ate/README.md): Installation and usage for `kubectl-ate`.
 * [Glossary](docs/glossary.md): Core terms (Actor, ActorTemplate, WorkerPool, Worker, ate-api-server, atenet, atelet, ateom) and how they relate.
 * [Observability Guide](docs/observability.md): Guide to actor logging, metrics, and distributed tracing.
@@ -216,7 +216,7 @@ We provide several sample applications demonstrating Agent Substrate's capabilit
 * `cmd/atelet`: A node-level DaemonSet that supervises physical worker pods, coordinates snapshotting, and manages state transfers.
 * `cmd/atecontroller`: A Kubernetes controller that reconciles WorkerPool and ActorTemplate custom resources.
 * `cmd/atenet`: A combined networking controller providing DNS, Envoy routing, and proxy sidecars.
-* `cmd/ateom-gvisor`: An interior-pod helper running inside sandboxed worker pods to execute `runsc` checkpoint and restore commands.
+* `cmd/ateom-gvisor`: An interior-pod helper running inside sandboxed worker pods to execute `runsc` checkpoint and restore commands, and to inject GPU devices (CDI) into actor containers on GPU pools.
 * `cmd/podcertcontroller`: A "polyfill" that provides Pod Certificate signers that
   will eventually ship in upstream Kubernetes (with different names).
 * `cmd/kubectl-ate`: A CLI tool for managing Agent Substrate resources. See its [README](cmd/kubectl-ate/README.md).

cmd/atecontroller/internal/controllers/workerpool_apply.go

@@ -61,6 +61,7 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool) *appsv1ac.Deployment
 
 	applyWorkerPoolPodTemplate(podSpecAC, containerAC, wp.Spec.Template)
 	maybeApplyMicroVMPodShape(podSpecAC, containerAC, wp.Spec.SandboxClass)
+	maybeApplyGPUPodShape(podSpecAC, containerAC, wp.Spec.Template, wp.Spec.SandboxClass)
 	podSpecAC.WithContainers(containerAC)
 
 	return appsv1ac.Deployment(wp.Name, wp.Namespace).
@@ -129,6 +130,58 @@ func maybeApplyMicroVMPodShape(
 		WithEffect(corev1.TaintEffectNoSchedule))
 }
 
+// nvidiaToolkitHostPath is where the NVIDIA container toolkit is installed on GPU
+// nodes and where we mount it into the worker pod. ateom-gvisor reads nvidia-ctk and
+// nvidia-cdi-hook from this same path (see its toolkitDir constant) — the two must
+// match.
+const nvidiaToolkitHostPath = "/usr/local/nvidia/toolkit"
+
+// maybeApplyGPUPodShape mounts the host NVIDIA container toolkit into the worker
+// pod when a gVisor pool requests a GPU. ateom-gvisor uses nvidia-ctk (generate)
+// and nvidia-cdi-hook (runtime CDI hooks) from this mount to inject the GPU into
+// the actor's gVisor sandbox. No-op for non-GPU pools and for non-gVisor classes:
+// the toolkit is specific to the CDI/nvproxy path, and micro-VM GPU would instead
+// be VFIO PCI passthrough into a guest that carries its own driver. An empty class
+// defaults to gVisor (WorkerPoolSpec kubebuilder default).
+func maybeApplyGPUPodShape(
+	podSpecAC *corev1ac.PodSpecApplyConfiguration,
+	containerAC *corev1ac.ContainerApplyConfiguration,
+	tmpl *atev1alpha1.WorkerPoolPodTemplate,
+	sandboxClass atev1alpha1.SandboxClass,
+) {
+	if sandboxClass != atev1alpha1.SandboxClassGvisor && sandboxClass != "" {
+		return
+	}
+	if !templateRequestsGPU(tmpl) {
+		return
+	}
+	containerAC.WithVolumeMounts(corev1ac.VolumeMount().
+		WithName("nvidia-toolkit").
+		WithMountPath(nvidiaToolkitHostPath).
+		WithReadOnly(true))
+	podSpecAC.WithVolumes(corev1ac.Volume().
+		WithName("nvidia-toolkit").
+		WithHostPath(corev1ac.HostPathVolumeSource().
+			WithPath(nvidiaToolkitHostPath).
+			WithType(corev1.HostPathDirectory)))
+}
+
+// templateRequestsGPU reports whether the pool template requests one or more
+// nvidia.com/gpu devices (limits or requests).
+func templateRequestsGPU(tmpl *atev1alpha1.WorkerPoolPodTemplate) bool {
+	if tmpl == nil || tmpl.Resources == nil {
+		return false
+	}
+	const gpu = corev1.ResourceName("nvidia.com/gpu")
+	if q, ok := tmpl.Resources.Limits[gpu]; ok && !q.IsZero() {
+		return true
+	}
+	if q, ok := tmpl.Resources.Requests[gpu]; ok && !q.IsZero() {
+		return true
+	}
+	return false
+}
+
 func applyWorkerPoolPodTemplate(
 	podSpecAC *corev1ac.PodSpecApplyConfiguration,
 	containerAC *corev1ac.ContainerApplyConfiguration,

cmd/atecontroller/internal/controllers/workerpool_apply_test.go

@@ -261,6 +261,86 @@ func TestMicroVMPodShape(t *testing.T) {
 	}
 }
 
+func TestGPUPoolMountsToolkit(t *testing.T) {
+	gpu := resource.MustParse("1")
+	wp := &atev1alpha1.WorkerPool{
+		ObjectMeta: metav1.ObjectMeta{Name: "wp", Namespace: "ns"},
+		Spec: atev1alpha1.WorkerPoolSpec{
+			AteomImage: "img",
+			Template: &atev1alpha1.WorkerPoolPodTemplate{
+				Resources: &corev1.ResourceRequirements{
+					Limits: corev1.ResourceList{"nvidia.com/gpu": gpu},
+				},
+			},
+		},
+	}
+	dep := buildDeploymentApplyConfig(wp)
+	pod := dep.Spec.Template.Spec
+
+	var found bool
+	for _, v := range pod.Volumes {
+		if v.Name != nil && *v.Name == "nvidia-toolkit" {
+			found = true
+			if v.HostPath == nil || *v.HostPath.Path != "/usr/local/nvidia/toolkit" {
+				t.Fatalf("nvidia-toolkit volume has wrong hostPath: %+v", v.HostPath)
+			}
+		}
+	}
+	if !found {
+		t.Fatal("expected nvidia-toolkit volume on a GPU pool")
+	}
+
+	var mounted bool
+	for _, c := range pod.Containers {
+		for _, m := range c.VolumeMounts {
+			if m.Name != nil && *m.Name == "nvidia-toolkit" && *m.MountPath == "/usr/local/nvidia/toolkit" {
+				mounted = true
+			}
+		}
+	}
+	if !mounted {
+		t.Fatal("expected nvidia-toolkit mount on the ateom container")
+	}
+}
+
+func TestNonGPUPoolHasNoToolkit(t *testing.T) {
+	wp := &atev1alpha1.WorkerPool{
+		ObjectMeta: metav1.ObjectMeta{Name: "wp", Namespace: "ns"},
+		Spec:       atev1alpha1.WorkerPoolSpec{AteomImage: "img"},
+	}
+	dep := buildDeploymentApplyConfig(wp)
+	for _, v := range dep.Spec.Template.Spec.Volumes {
+		if v.Name != nil && *v.Name == "nvidia-toolkit" {
+			t.Fatal("non-GPU pool must not mount the toolkit")
+		}
+	}
+}
+
+// TestGPUMicroVMPoolHasNoToolkit asserts the NVIDIA toolkit mount is gVisor-only:
+// a micro-VM pool that requests a GPU must not get it (kata GPU is VFIO
+// passthrough into the guest, which needs no host toolkit).
+func TestGPUMicroVMPoolHasNoToolkit(t *testing.T) {
+	gpu := resource.MustParse("1")
+	wp := &atev1alpha1.WorkerPool{
+		ObjectMeta: metav1.ObjectMeta{Name: "wp", Namespace: "ns"},
+		Spec: atev1alpha1.WorkerPoolSpec{
+			AteomImage:   "img",
+			SandboxClass: atev1alpha1.SandboxClassMicroVM,
+			Template: &atev1alpha1.WorkerPoolPodTemplate{
+				Resources: &corev1.ResourceRequirements{
+					Limits: corev1.ResourceList{"nvidia.com/gpu": gpu},
+				},
+			},
+		},
+	}
+	dep := buildDeploymentApplyConfig(wp)
+	for _, v := range dep.Spec.Template.Spec.Volumes {
+		if v.Name != nil && *v.Name == "nvidia-toolkit" {
+			t.Fatal("micro-VM pool must not mount the NVIDIA toolkit even when it requests a GPU")
+		}
+	}
+}
+
 func testWorkerPoolApplyConfig(tmpl *atev1alpha1.WorkerPoolPodTemplate) *atev1alpha1.WorkerPool {
 	return &atev1alpha1.WorkerPool{
 		ObjectMeta: metav1.ObjectMeta{Name: "pool", Namespace: "default", UID: "uid"},