From 764b0d900ee5e1aee46b288a9fd9abee7bcc7001 Mon Sep 17 00:00:00 2001 From: Mark Spicer Date: Thu, 23 Jul 2026 12:44:19 -0400 Subject: [PATCH 1/2] fix(gpu): force disable eBPF probes This commit forces the disablement of the eBPF probes when privileged mode is enabled. --- internal/controller/datadogagent/feature/gpu/envvar.go | 3 +++ internal/controller/datadogagent/feature/gpu/feature.go | 7 +++++++ .../controller/datadogagent/feature/gpu/feature_test.go | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/internal/controller/datadogagent/feature/gpu/envvar.go b/internal/controller/datadogagent/feature/gpu/envvar.go index 46940b287b..8213b2f35a 100644 --- a/internal/controller/datadogagent/feature/gpu/envvar.go +++ b/internal/controller/datadogagent/feature/gpu/envvar.go @@ -17,3 +17,6 @@ const NVIDIAVisibleDevicesEnvVar = "NVIDIA_VISIBLE_DEVICES" // DDPatchCgroupPermissionsEnvVar is the name of the system-probe gpu_monitoring module cgroup permissions patch knob const DDPatchCgroupPermissionsEnvVar = "DD_GPU_MONITORING_CONFIGURE_CGROUP_PERMS" + +// DDEnableEBPFProbesEnvVar is the name of the system-probe gpu_monitoring module eBPF probes enablement knob +const DDEnableEBPFProbesEnvVar = "DD_GPU_MONITORING_ENABLE_EBPF_PROBES" diff --git a/internal/controller/datadogagent/feature/gpu/feature.go b/internal/controller/datadogagent/feature/gpu/feature.go index ad2a949a1d..498c8f8690 100644 --- a/internal/controller/datadogagent/feature/gpu/feature.go +++ b/internal/controller/datadogagent/feature/gpu/feature.go @@ -127,6 +127,13 @@ func configureSystemProbe(managers feature.PodTemplateManagers) { // add the env var to the core agent as well, to prevent config mismatches in runtime managers.EnvVar().AddEnvVarToContainer(apicommon.CoreAgentContainerName, enableSPEnvVar) + // In privileged mode the eBPF probes are disabled, as GPU monitoring relies on + // the privileged host access rather than the eBPF probes. + managers.EnvVar().AddEnvVarToContainer(apicommon.SystemProbeContainerName, &corev1.EnvVar{ + Name: DDEnableEBPFProbesEnvVar, + Value: "false", + }) + // annotations managers.Annotation().AddAnnotation(common.SystemProbeAppArmorAnnotationKey, common.SystemProbeAppArmorAnnotationValue) diff --git a/internal/controller/datadogagent/feature/gpu/feature_test.go b/internal/controller/datadogagent/feature/gpu/feature_test.go index 6824e6640d..6498e44f1c 100644 --- a/internal/controller/datadogagent/feature/gpu/feature_test.go +++ b/internal/controller/datadogagent/feature/gpu/feature_test.go @@ -193,6 +193,10 @@ func Test_GPUMonitoringFeature_Configure(t *testing.T) { Name: DDEnableGPUProbeEnvVar, Value: "true", }, + { + Name: DDEnableEBPFProbesEnvVar, + Value: "false", + }, { Name: NVIDIAVisibleDevicesEnvVar, Value: "all", From 310273ccddc596f54225a20f80d89b8be23ba0cb Mon Sep 17 00:00:00 2001 From: Mark Spicer Date: Thu, 23 Jul 2026 13:38:22 -0400 Subject: [PATCH 2/2] fix(gpu): disable eBPF probes on core agent too Add DD_GPU_MONITORING_ENABLE_EBPF_PROBES=false to the core agent container as well as system-probe, so both containers agree the eBPF probes are disabled. Previously only system-probe received the flag, while the core agent still saw GPU monitoring enabled and would poll a system-probe module whose probes were off, causing recurring refresh warnings and empty process metrics. Co-Authored-By: Claude Opus 4.8 --- .../controller/datadogagent/feature/gpu/feature.go | 12 ++++++++++-- .../datadogagent/feature/gpu/feature_test.go | 4 ++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/internal/controller/datadogagent/feature/gpu/feature.go b/internal/controller/datadogagent/feature/gpu/feature.go index 498c8f8690..99f34d9a2c 100644 --- a/internal/controller/datadogagent/feature/gpu/feature.go +++ b/internal/controller/datadogagent/feature/gpu/feature.go @@ -129,10 +129,18 @@ func configureSystemProbe(managers feature.PodTemplateManagers) { // In privileged mode the eBPF probes are disabled, as GPU monitoring relies on // the privileged host access rather than the eBPF probes. - managers.EnvVar().AddEnvVarToContainer(apicommon.SystemProbeContainerName, &corev1.EnvVar{ + disableEBPFProbesEnvVar := &corev1.EnvVar{ Name: DDEnableEBPFProbesEnvVar, Value: "false", - }) + } + + // disable the eBPF probes in system-probe + managers.EnvVar().AddEnvVarToContainer(apicommon.SystemProbeContainerName, disableEBPFProbesEnvVar) + + // add the env var to the core agent as well, so both containers agree that the + // eBPF probes are disabled and the core GPU check does not poll a system-probe + // module that has the probes turned off + managers.EnvVar().AddEnvVarToContainer(apicommon.CoreAgentContainerName, disableEBPFProbesEnvVar) // annotations managers.Annotation().AddAnnotation(common.SystemProbeAppArmorAnnotationKey, common.SystemProbeAppArmorAnnotationValue) diff --git a/internal/controller/datadogagent/feature/gpu/feature_test.go b/internal/controller/datadogagent/feature/gpu/feature_test.go index 6498e44f1c..3ae8a55819 100644 --- a/internal/controller/datadogagent/feature/gpu/feature_test.go +++ b/internal/controller/datadogagent/feature/gpu/feature_test.go @@ -220,6 +220,10 @@ func Test_GPUMonitoringFeature_Configure(t *testing.T) { Name: DDEnableGPUProbeEnvVar, Value: "true", }, + { + Name: DDEnableEBPFProbesEnvVar, + Value: "false", + }, { Name: common.DDSystemProbeSocket, Value: common.DefaultSystemProbeSocketPath,