From 59585273161cdfd3fae9800e6f2ad49b205d6d46 Mon Sep 17 00:00:00 2001 From: "ali.benabdallah" Date: Wed, 22 Jul 2026 14:08:15 +0200 Subject: [PATCH 01/16] Add resource-aware fallback for Agent surge --- go.mod | 2 +- .../datadogagent/controller_v2_test.go | 2 + .../datadogagentinternal/controller.go | 4 +- .../controller_reconcile_agent.go | 16 +- .../datadogagentinternal/resource_fallback.go | 701 ++++++++++++++++++ .../resource_fallback_test.go | 387 ++++++++++ .../datadogagentinternal_controller.go | 105 ++- .../datadogagentinternal_controller_test.go | 75 ++ .../controller/testutils/renderer/renderer.go | 2 +- pkg/config/config.go | 23 +- pkg/config/config_test.go | 9 +- 11 files changed, 1293 insertions(+), 33 deletions(-) create mode 100644 internal/controller/datadogagentinternal/resource_fallback.go create mode 100644 internal/controller/datadogagentinternal/resource_fallback_test.go create mode 100644 internal/controller/datadogagentinternal_controller_test.go diff --git a/go.mod b/go.mod index 84d3c3b9d9..db143f08f6 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( k8s.io/apimachinery v0.35.3 k8s.io/cli-runtime v0.35.3 k8s.io/client-go v0.35.3 + k8s.io/component-helpers v0.35.3 k8s.io/klog/v2 v2.140.0 k8s.io/kube-aggregator v0.35.3 k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e @@ -287,7 +288,6 @@ require ( k8s.io/apiserver v0.35.3 // indirect k8s.io/cloud-provider v0.35.0 // indirect k8s.io/component-base v0.35.3 // indirect - k8s.io/component-helpers v0.35.3 // indirect k8s.io/csi-translation-lib v0.35.0 // indirect k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b // indirect modernc.org/sqlite v1.34.1 // indirect diff --git a/internal/controller/datadogagent/controller_v2_test.go b/internal/controller/datadogagent/controller_v2_test.go index bfe508e2fe..3903249ce3 100644 --- a/internal/controller/datadogagent/controller_v2_test.go +++ b/internal/controller/datadogagent/controller_v2_test.go @@ -121,6 +121,7 @@ func runDDAReconcilerTest(t *testing.T, tt testCase, opts ReconcilerOptions) { ri := datadogagentinternal.NewReconciler( ddaiReconcilerOptionsFromDDA(opts), c, + c, kubernetes.PlatformInfo{}, s, recorder, @@ -194,6 +195,7 @@ func runFullReconcilerTest(t *testing.T, tt testCase, opts ReconcilerOptions) { ri := datadogagentinternal.NewReconciler( ddaiReconcilerOptionsFromDDA(opts), c, + c, kubernetes.PlatformInfo{}, s, recorder, diff --git a/internal/controller/datadogagentinternal/controller.go b/internal/controller/datadogagentinternal/controller.go index ff192c0d7e..02ad774325 100644 --- a/internal/controller/datadogagentinternal/controller.go +++ b/internal/controller/datadogagentinternal/controller.go @@ -74,6 +74,7 @@ type ReconcilerOptions struct { type Reconciler struct { options ReconcilerOptions client client.Client + apiReader client.Reader platformInfo kubernetes.PlatformInfo scheme *runtime.Scheme recorder record.EventRecorder @@ -90,10 +91,11 @@ func (r *Reconciler) initializeComponentRegistry() { } // NewReconciler returns a reconciler for DatadogAgent -func NewReconciler(options ReconcilerOptions, client client.Client, platformInfo kubernetes.PlatformInfo, scheme *runtime.Scheme, recorder record.EventRecorder, metricForwardersMgr datadog.MetricsForwardersManager) *Reconciler { +func NewReconciler(options ReconcilerOptions, client client.Client, apiReader client.Reader, platformInfo kubernetes.PlatformInfo, scheme *runtime.Scheme, recorder record.EventRecorder, metricForwardersMgr datadog.MetricsForwardersManager) *Reconciler { r := &Reconciler{ options: options, client: client, + apiReader: apiReader, platformInfo: platformInfo, scheme: scheme, recorder: recorder, diff --git a/internal/controller/datadogagentinternal/controller_reconcile_agent.go b/internal/controller/datadogagentinternal/controller_reconcile_agent.go index 33fd4d44ab..b936c40b30 100644 --- a/internal/controller/datadogagentinternal/controller_reconcile_agent.go +++ b/internal/controller/datadogagentinternal/controller_reconcile_agent.go @@ -36,6 +36,7 @@ import ( func (r *Reconciler) reconcileV2Agent(ctx context.Context, requiredComponents feature.RequiredComponents, features []feature.Feature, ddai *datadoghqv1alpha1.DatadogAgentInternal, resourcesManager feature.ResourceManagers, newStatus *datadoghqv1alpha1.DatadogAgentInternalStatus, provider string) (reconcile.Result, error) { var result reconcile.Result + var err error var eds *edsv1alpha1.ExtendedDaemonSet var daemonset *appsv1.DaemonSet var podManagers feature.PodTemplateManagers @@ -190,7 +191,20 @@ func (r *Reconciler) reconcileV2Agent(ctx context.Context, requiredComponents fe return reconcile.Result{}, nil } - return r.createOrUpdateDaemonset(ctx, ddai, daemonset, newStatus, updateDSStatusV2WithAgent) + fallbackBudget := resourceFallbackBudget(ddai, &r.options.ExtendedDaemonsetOptions) + fallbackEnabled := configureResourceFallback(daemonset, fallbackBudget) + result, err = r.createOrUpdateDaemonset(ctx, ddai, daemonset, newStatus, updateDSStatusV2WithAgent) + if err != nil || !fallbackEnabled { + return result, err + } + fallbackResult, err := r.reconcileResourceFallback(ctx, ddai, daemonset, fallbackBudget) + if err != nil { + return reconcile.Result{}, err + } + if fallbackResult.RequeueAfter > 0 && (result.RequeueAfter == 0 || fallbackResult.RequeueAfter < result.RequeueAfter) { + result.RequeueAfter = fallbackResult.RequeueAfter + } + return result, nil } func updateDSStatusV2WithAgent(dsName string, ds *appsv1.DaemonSet, newStatus *datadoghqv1alpha1.DatadogAgentInternalStatus, updateTime metav1.Time, status metav1.ConditionStatus, reason, message string) { diff --git a/internal/controller/datadogagentinternal/resource_fallback.go b/internal/controller/datadogagentinternal/resource_fallback.go new file mode 100644 index 0000000000..c9395c4d13 --- /dev/null +++ b/internal/controller/datadogagentinternal/resource_fallback.go @@ -0,0 +1,701 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// Copyright 2016-present Datadog, Inc. + +package datadogagentinternal + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strconv" + "strings" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + resourcehelper "k8s.io/component-helpers/resource" + "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" + datadoghqv2alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v2alpha1" + componentagent "github.com/DataDog/datadog-operator/internal/controller/datadogagent/component/agent" +) + +const ( + resourceFallbackOldPodAnnotation = "agent.datadoghq.com/resource-fallback-old-pod-uid" + apiPodNodeNameField = "spec.nodeName" + defaultFallbackMaxUnavailable = 1 +) + +// configureResourceFallback keeps surge opt-in. When it is requested, the +// existing maxUnavailable setting becomes both the surge limit and the +// Operator's resource-pressure fallback budget. The emitted maxUnavailable is +// intentionally 0 so the native DaemonSet controller never proactively +// deletes an old Pod; only the resource-proven fallback below may do that. +func configureResourceFallback(ds *appsv1.DaemonSet, budget intstr.IntOrString) bool { + strategy := &ds.Spec.UpdateStrategy + if strategy.Type != "" && strategy.Type != appsv1.RollingUpdateDaemonSetStrategyType { + return false + } + if strategy.RollingUpdate == nil || !positiveIntOrPercent(strategy.RollingUpdate.MaxSurge) { + return false + } + if _, err := intstr.GetScaledValueFromIntOrPercent(&budget, 100, true); err != nil { + return false + } + + strategy.Type = appsv1.RollingUpdateDaemonSetStrategyType + zero := intstr.FromInt(0) + strategy.RollingUpdate.MaxUnavailable = &zero + if positiveIntOrPercent(&budget) { + surge := budget + strategy.RollingUpdate.MaxSurge = &surge + return true + } + return false +} + +func positiveIntOrPercent(value *intstr.IntOrString) bool { + if value == nil { + return false + } + scaled, err := intstr.GetScaledValueFromIntOrPercent(value, 100, true) + return err == nil && scaled > 0 +} + +func resourceFallbackBudget(ddai *datadoghqv1alpha1.DatadogAgentInternal, options *componentagent.ExtendedDaemonsetOptions) intstr.IntOrString { + if override, ok := ddai.Spec.Override[datadoghqv2alpha1.NodeAgentComponentName]; ok && override != nil && override.UpdateStrategy != nil && override.UpdateStrategy.RollingUpdate != nil && override.UpdateStrategy.RollingUpdate.MaxUnavailable != nil { + return *override.UpdateStrategy.RollingUpdate.MaxUnavailable + } + if options != nil && options.MaxPodUnavailable != "" { + return intstr.Parse(options.MaxPodUnavailable) + } + return intstr.FromInt(defaultFallbackMaxUnavailable) +} + +type resourceShortage struct { + cpu bool + memory bool +} + +type fallbackCandidate struct { + pending *corev1.Pod + old *corev1.Pod + nodeName string + shortage resourceShortage + reserved bool +} + +func (r *Reconciler) reconcileResourceFallback(ctx context.Context, ddai *datadoghqv1alpha1.DatadogAgentInternal, expectedDS *appsv1.DaemonSet, budgetValue intstr.IntOrString) (reconcile.Result, error) { + reader := r.apiReader + if reader == nil { + reader = r.client + } + + ds := &appsv1.DaemonSet{} + key := client.ObjectKeyFromObject(expectedDS) + if err := reader.Get(ctx, key, ds); err != nil { + if apierrors.IsNotFound(err) { + return reconcile.Result{}, nil + } + return reconcile.Result{}, fmt.Errorf("get Agent DaemonSet for resource fallback: %w", err) + } + if !daemonSetControlledByDDAI(ds, ddai) || !resourceFallbackDaemonSetEligible(ds) { + return reconcile.Result{}, nil + } + + desired := int(ds.Status.DesiredNumberScheduled) + budget, err := intstr.GetScaledValueFromIntOrPercent(&budgetValue, desired, true) + if err != nil || budget <= 0 { + return reconcile.Result{}, nil + } + + currentRevision, err := currentDaemonSetRevision(ctx, reader, ds) + if err != nil { + return reconcile.Result{}, err + } + if currentRevision == "" { + return reconcile.Result{}, nil + } + + pods, err := daemonSetPods(ctx, reader, ds) + if err != nil { + return reconcile.Result{}, err + } + consumed := consumedFallbackBudget(ds, pods, currentRevision, time.Now()) + candidates := fallbackCandidates(ds, pods, currentRevision, time.Now()) + if consumed > budget { + return reconcile.Result{}, nil + } + + for _, candidate := range candidates { + if !candidate.reserved && consumed >= budget { + break + } + + if !candidate.reserved { + liveCandidate, err := r.revalidateFallbackCandidate(ctx, reader, ds, candidate, currentRevision, false) + if err != nil { + return reconcile.Result{}, err + } + if liveCandidate == nil { + continue + } + candidate = *liveCandidate + base := candidate.pending.DeepCopy() + patched := candidate.pending.DeepCopy() + if patched.Annotations == nil { + patched.Annotations = map[string]string{} + } + patched.Annotations[resourceFallbackOldPodAnnotation] = string(candidate.old.UID) + if err := r.client.Patch(ctx, patched, client.MergeFrom(base)); err != nil { + return reconcile.Result{}, fmt.Errorf("reserve Agent resource fallback for Pod %s/%s: %w", patched.Namespace, patched.Name, err) + } + candidate.pending = patched + candidate.reserved = true + consumed++ + } + + liveCandidate, err := r.revalidateFallbackCandidate(ctx, reader, ds, candidate, currentRevision, true) + if err != nil { + return reconcile.Result{}, err + } + if liveCandidate == nil { + continue + } + withinBudget, err := fallbackBudgetWithinLimit(ctx, reader, ds, budgetValue, currentRevision) + if err != nil { + return reconcile.Result{}, err + } + if !withinBudget { + return reconcile.Result{RequeueAfter: time.Second}, nil + } + + uid := liveCandidate.old.UID + if err := r.client.Delete(ctx, liveCandidate.old, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("delete old Agent Pod %s/%s for resource fallback: %w", liveCandidate.old.Namespace, liveCandidate.old.Name, err) + } + + logger := ctrl.LoggerFrom(ctx).WithValues("daemonset", ds.Name, "node", liveCandidate.nodeName, "oldPod", liveCandidate.old.Name, "replacementPod", liveCandidate.pending.Name) + logger.Info("Deleted old Agent Pod after proving the surged replacement was blocked only by node CPU or memory") + if r.recorder != nil { + r.recorder.Eventf(ddai, corev1.EventTypeWarning, "AgentResourceFallback", "Deleted old Agent Pod %s on node %s because replacement %s could not fit alongside it", liveCandidate.old.Name, liveCandidate.nodeName, liveCandidate.pending.Name) + } + return reconcile.Result{RequeueAfter: time.Second}, nil + } + + return reconcile.Result{}, nil +} + +func fallbackBudgetWithinLimit(ctx context.Context, reader client.Reader, expectedDS *appsv1.DaemonSet, budgetValue intstr.IntOrString, expectedRevision string) (bool, error) { + liveDS := &appsv1.DaemonSet{} + if err := reader.Get(ctx, client.ObjectKeyFromObject(expectedDS), liveDS); err != nil { + return false, client.IgnoreNotFound(err) + } + if liveDS.UID != expectedDS.UID || liveDS.Generation != expectedDS.Generation || !resourceFallbackDaemonSetEligible(liveDS) { + return false, nil + } + revision, err := currentDaemonSetRevision(ctx, reader, liveDS) + if err != nil || revision != expectedRevision { + return false, err + } + pods, err := daemonSetPods(ctx, reader, liveDS) + if err != nil { + return false, err + } + budget, err := intstr.GetScaledValueFromIntOrPercent(&budgetValue, int(liveDS.Status.DesiredNumberScheduled), true) + if err != nil || budget <= 0 { + return false, err + } + return consumedFallbackBudget(liveDS, pods, revision, time.Now()) <= budget, nil +} + +func daemonSetControlledByDDAI(ds *appsv1.DaemonSet, ddai *datadoghqv1alpha1.DatadogAgentInternal) bool { + owner := metav1.GetControllerOf(ds) + return owner != nil && owner.APIVersion == datadoghqv1alpha1.GroupVersion.String() && owner.Kind == "DatadogAgentInternal" && owner.UID == ddai.UID +} + +func resourceFallbackDaemonSetEligible(ds *appsv1.DaemonSet) bool { + if ds.DeletionTimestamp != nil || ds.Status.DesiredNumberScheduled <= 0 || ds.Status.ObservedGeneration != ds.Generation { + return false + } + if ds.Spec.UpdateStrategy.Type != appsv1.RollingUpdateDaemonSetStrategyType || ds.Spec.UpdateStrategy.RollingUpdate == nil || !positiveIntOrPercent(ds.Spec.UpdateStrategy.RollingUpdate.MaxSurge) { + return false + } + return true +} + +func currentDaemonSetRevision(ctx context.Context, reader client.Reader, ds *appsv1.DaemonSet) (string, error) { + revisions := &appsv1.ControllerRevisionList{} + if err := reader.List(ctx, revisions, client.InNamespace(ds.Namespace)); err != nil { + return "", fmt.Errorf("list revisions for Agent DaemonSet %s/%s: %w", ds.Namespace, ds.Name, err) + } + var current *appsv1.ControllerRevision + for i := range revisions.Items { + revision := &revisions.Items[i] + if !controlledByUID(revision, ds.UID) { + continue + } + matches, err := controllerRevisionMatchesTemplate(revision, &ds.Spec.Template) + if err != nil { + return "", fmt.Errorf("decode revision %s for Agent DaemonSet %s/%s: %w", revision.Name, ds.Namespace, ds.Name, err) + } + if matches && (current == nil || revision.Revision > current.Revision) { + current = revision + } + } + if current == nil { + return "", nil + } + return current.Labels[appsv1.DefaultDaemonSetUniqueLabelKey], nil +} + +func controllerRevisionMatchesTemplate(revision *appsv1.ControllerRevision, template *corev1.PodTemplateSpec) (bool, error) { + var patch struct { + Spec struct { + Template corev1.PodTemplateSpec `json:"template"` + } `json:"spec"` + } + if err := json.Unmarshal(revision.Data.Raw, &patch); err != nil { + return false, err + } + return apiequality.Semantic.DeepEqual(patch.Spec.Template, *template), nil +} + +func daemonSetPods(ctx context.Context, reader client.Reader, ds *appsv1.DaemonSet) ([]corev1.Pod, error) { + selector, err := metav1.LabelSelectorAsSelector(ds.Spec.Selector) + if err != nil { + return nil, fmt.Errorf("build selector for Agent DaemonSet %s/%s: %w", ds.Namespace, ds.Name, err) + } + list := &corev1.PodList{} + if err := reader.List(ctx, list, client.InNamespace(ds.Namespace), client.MatchingLabelsSelector{Selector: selector}); err != nil { + return nil, fmt.Errorf("list Pods for Agent DaemonSet %s/%s: %w", ds.Namespace, ds.Name, err) + } + result := make([]corev1.Pod, 0, len(list.Items)) + for i := range list.Items { + if controlledByUID(&list.Items[i], ds.UID) { + result = append(result, list.Items[i]) + } + } + return result, nil +} + +func controlledByUID(obj metav1.Object, uid types.UID) bool { + owner := metav1.GetControllerOf(obj) + return owner != nil && owner.UID == uid +} + +func fallbackCandidates(ds *appsv1.DaemonSet, pods []corev1.Pod, currentRevision string, now time.Time) []fallbackCandidate { + result := make([]fallbackCandidate, 0) + for i := range pods { + pending := &pods[i] + shortage, ok := resourceOnlyUnschedulable(pending) + if !ok || !resourceFallbackSchedulingShapeSafe(pending) || pending.DeletionTimestamp != nil || pending.Spec.NodeName != "" || pending.Status.NominatedNodeName != "" || pending.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] != currentRevision { + continue + } + nodeName, ok := targetNodeFromDaemonSetAffinity(pending) + if !ok { + continue + } + + var oldPods []*corev1.Pod + for j := range pods { + old := &pods[j] + oldRevision := old.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] + if old.Spec.NodeName == nodeName && oldRevision != "" && oldRevision != currentRevision && podAvailable(old, ds.Spec.MinReadySeconds, now) { + oldPods = append(oldPods, old) + } + } + if len(oldPods) != 1 { + continue + } + + reservedUID := pending.Annotations[resourceFallbackOldPodAnnotation] + if reservedUID != "" && reservedUID != string(oldPods[0].UID) { + continue + } + result = append(result, fallbackCandidate{pending: pending, old: oldPods[0], nodeName: nodeName, shortage: shortage, reserved: reservedUID != ""}) + } + sort.Slice(result, func(i, j int) bool { + if result[i].reserved != result[j].reserved { + return result[i].reserved + } + return result[i].nodeName < result[j].nodeName + }) + return result +} + +func resourceOnlyUnschedulable(pod *corev1.Pod) (resourceShortage, bool) { + condition := scheduledCondition(pod) + if condition == nil || condition.Status != corev1.ConditionFalse || condition.Reason != corev1.PodReasonUnschedulable { + return resourceShortage{}, false + } + + primary := condition.Message + lower := strings.ToLower(primary) + if i := strings.Index(lower, "preemption:"); i >= 0 { + primary = primary[:i] + lower = lower[:i] + } + if i := strings.Index(lower, "nodes are available:"); i >= 0 { + primary = primary[i+len("nodes are available:"):] + } + primary = strings.TrimSuffix(strings.TrimSpace(primary), ".") + if primary == "" { + return resourceShortage{}, false + } + + var shortage resourceShortage + for _, reason := range strings.Split(primary, ", ") { + fields := strings.Fields(strings.ToLower(strings.TrimSpace(reason))) + if len(fields) < 2 { + return resourceShortage{}, false + } + if _, err := strconv.Atoi(fields[0]); err != nil { + return resourceShortage{}, false + } + normalized := strings.Join(fields[1:], " ") + switch normalized { + case "insufficient cpu": + shortage.cpu = true + case "insufficient memory": + shortage.memory = true + case "node(s) didn't match pod's node affinity/selector", "node(s) didn't satisfy plugin(s) [nodeaffinity]": + // Expected for every non-target node because DaemonSet surge Pods are + // pinned through required node affinity. + default: + return resourceShortage{}, false + } + } + return shortage, shortage.cpu || shortage.memory +} + +func scheduledCondition(pod *corev1.Pod) *corev1.PodCondition { + for i := range pod.Status.Conditions { + if pod.Status.Conditions[i].Type == corev1.PodScheduled { + return &pod.Status.Conditions[i] + } + } + return nil +} + +func targetNodeFromDaemonSetAffinity(pod *corev1.Pod) (string, bool) { + if pod.Spec.Affinity == nil || pod.Spec.Affinity.NodeAffinity == nil || pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + return "", false + } + terms := pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + if len(terms) == 0 { + return "", false + } + var target string + for _, term := range terms { + termTarget := "" + for _, requirement := range term.MatchFields { + if requirement.Key != metav1.ObjectNameField { + continue + } + if requirement.Operator != corev1.NodeSelectorOpIn || len(requirement.Values) != 1 || requirement.Values[0] == "" || termTarget != "" { + return "", false + } + termTarget = requirement.Values[0] + } + if termTarget == "" || (target != "" && termTarget != target) { + return "", false + } + target = termTarget + } + return target, target != "" +} + +// resourceFallbackSchedulingShapeSafe rejects Pod-declared constraints whose +// scheduler failure could be masked by a simultaneous CPU or memory shortage. +// Cluster-specific plugins configured under the default scheduler name are not +// visible through the Pod API and must be excluded operationally. +func resourceFallbackSchedulingShapeSafe(pod *corev1.Pod) bool { + if pod.Spec.SchedulerName != "" && pod.Spec.SchedulerName != corev1.DefaultSchedulerName { + return false + } + if pod.Spec.RuntimeClassName != nil || len(pod.Spec.TopologySpreadConstraints) > 0 { + return false + } + if pod.Spec.Affinity != nil && (pod.Spec.Affinity.PodAffinity != nil || pod.Spec.Affinity.PodAntiAffinity != nil) { + return false + } + for _, container := range append(append([]corev1.Container{}, pod.Spec.InitContainers...), pod.Spec.Containers...) { + for _, port := range container.Ports { + if port.HostPort != 0 { + return false + } + } + } + for _, volume := range pod.Spec.Volumes { + source := volume.VolumeSource + if source.EmptyDir == nil && source.HostPath == nil && source.ConfigMap == nil && source.Secret == nil && source.DownwardAPI == nil && source.Projected == nil { + return false + } + } + return true +} + +func consumedFallbackBudget(ds *appsv1.DaemonSet, pods []corev1.Pod, currentRevision string, now time.Time) int { + availableByNode := map[string]bool{} + knownNodes := map[string]bool{} + for i := range pods { + pod := &pods[i] + nodeName := pod.Spec.NodeName + if nodeName == "" { + nodeName, _ = targetNodeFromDaemonSetAffinity(pod) + } + if nodeName == "" { + continue + } + knownNodes[nodeName] = true + if podAvailable(pod, ds.Spec.MinReadySeconds, now) { + availableByNode[nodeName] = true + } + } + + reservations := 0 + reservedUnavailable := 0 + for i := range pods { + pod := &pods[i] + if pod.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] != currentRevision || pod.Annotations[resourceFallbackOldPodAnnotation] == "" || podAvailable(pod, ds.Spec.MinReadySeconds, now) { + continue + } + nodeName := pod.Spec.NodeName + if nodeName == "" { + nodeName, _ = targetNodeFromDaemonSetAffinity(pod) + } + reservations++ + if nodeName == "" || !availableByNode[nodeName] { + reservedUnavailable++ + } + } + + liveUnavailable := 0 + for nodeName := range knownNodes { + if !availableByNode[nodeName] { + liveUnavailable++ + } + } + if missingNodes := int(ds.Status.DesiredNumberScheduled) - len(knownNodes); missingNodes > 0 { + liveUnavailable += missingNodes + } + + statusUnavailable := int(ds.Status.NumberUnavailable) + statusBeyondLive := max(0, statusUnavailable-liveUnavailable) + return reservations + liveUnavailable - min(liveUnavailable, reservedUnavailable) + statusBeyondLive +} + +func podAvailable(pod *corev1.Pod, minReadySeconds int32, now time.Time) bool { + if pod.DeletionTimestamp != nil || pod.Status.Phase != corev1.PodRunning { + return false + } + for i := range pod.Status.Conditions { + condition := &pod.Status.Conditions[i] + if condition.Type != corev1.PodReady || condition.Status != corev1.ConditionTrue { + continue + } + return minReadySeconds == 0 || !condition.LastTransitionTime.IsZero() && condition.LastTransitionTime.Add(time.Duration(minReadySeconds)*time.Second).Before(now) + } + return false +} + +func (r *Reconciler) revalidateFallbackCandidate(ctx context.Context, reader client.Reader, expectedDS *appsv1.DaemonSet, candidate fallbackCandidate, expectedRevision string, requireReservation bool) (*fallbackCandidate, error) { + liveDS := &appsv1.DaemonSet{} + if err := reader.Get(ctx, client.ObjectKeyFromObject(expectedDS), liveDS); err != nil { + return nil, client.IgnoreNotFound(err) + } + if liveDS.UID != expectedDS.UID || liveDS.Generation != expectedDS.Generation || !resourceFallbackDaemonSetEligible(liveDS) { + return nil, nil + } + liveRevision, err := currentDaemonSetRevision(ctx, reader, liveDS) + if err != nil || liveRevision != expectedRevision { + return nil, err + } + + pending := &corev1.Pod{} + if err := reader.Get(ctx, client.ObjectKeyFromObject(candidate.pending), pending); err != nil { + return nil, client.IgnoreNotFound(err) + } + old := &corev1.Pod{} + if err := reader.Get(ctx, client.ObjectKeyFromObject(candidate.old), old); err != nil { + return nil, client.IgnoreNotFound(err) + } + if !controlledByUID(pending, liveDS.UID) || !controlledByUID(old, liveDS.UID) || pending.UID != candidate.pending.UID || old.UID != candidate.old.UID { + return nil, nil + } + shortage, ok := resourceOnlyUnschedulable(pending) + nodeName, targetOK := targetNodeFromDaemonSetAffinity(pending) + reservation := pending.Annotations[resourceFallbackOldPodAnnotation] + if !ok || !resourceFallbackSchedulingShapeSafe(pending) || !targetOK || nodeName != candidate.nodeName || pending.Spec.NodeName != "" || pending.Status.NominatedNodeName != "" || pending.DeletionTimestamp != nil || pending.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] != liveRevision || requireReservation && reservation != string(old.UID) || reservation != "" && reservation != string(old.UID) { + return nil, nil + } + if old.Spec.NodeName != nodeName || old.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] == "" || old.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] == liveRevision || !podAvailable(old, liveDS.Spec.MinReadySeconds, time.Now()) { + return nil, nil + } + + node := &corev1.Node{} + if err := reader.Get(ctx, client.ObjectKey{Name: nodeName}, node); err != nil { + return nil, client.IgnoreNotFound(err) + } + if !nodeReadyForResourceFallback(node) { + return nil, nil + } + matches, err := nodeaffinity.GetRequiredNodeAffinity(pending).Match(node) + if err != nil || !matches { + return nil, err + } + nodePods := &corev1.PodList{} + if err := reader.List(ctx, nodePods, client.MatchingFields{apiPodNodeNameField: nodeName}); err != nil { + return nil, fmt.Errorf("list Pods on node %s for Agent resource fallback: %w", nodeName, err) + } + if !resourceFitAfterOldPodRemoval(node, nodePods.Items, pending, old, shortage) { + return nil, nil + } + + return &fallbackCandidate{pending: pending, old: old, nodeName: nodeName, shortage: shortage, reserved: reservation != ""}, nil +} + +func nodeReadyForResourceFallback(node *corev1.Node) bool { + if node.Spec.Unschedulable || node.DeletionTimestamp != nil { + return false + } + ready := false + pressureHealthy := map[corev1.NodeConditionType]bool{ + corev1.NodeMemoryPressure: false, + corev1.NodeDiskPressure: false, + corev1.NodePIDPressure: false, + } + for i := range node.Status.Conditions { + condition := node.Status.Conditions[i] + switch condition.Type { + case corev1.NodeReady: + ready = condition.Status == corev1.ConditionTrue + case corev1.NodeMemoryPressure, corev1.NodeDiskPressure, corev1.NodePIDPressure: + if condition.Status != corev1.ConditionFalse { + return false + } + pressureHealthy[condition.Type] = true + case corev1.NodeNetworkUnavailable: + if condition.Status != corev1.ConditionFalse { + return false + } + } + } + return ready && pressureHealthy[corev1.NodeMemoryPressure] && pressureHealthy[corev1.NodeDiskPressure] && pressureHealthy[corev1.NodePIDPressure] +} + +func resourceFitAfterOldPodRemoval(node *corev1.Node, nodePods []corev1.Pod, replacement, old *corev1.Pod, shortage resourceShortage) bool { + if len(replacement.Spec.ResourceClaims) > 0 { + return false + } + used := corev1.ResourceList{} + oldFound := false + podCount := int64(0) + for i := range nodePods { + pod := &nodePods[i] + if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { + continue + } + podCount++ + addResources(used, schedulerPodRequests(pod)) + if pod.UID == old.UID { + oldFound = true + } + } + if !oldFound { + return false + } + + replacementRequests := schedulerPodRequests(replacement) + oldRequests := schedulerPodRequests(old) + before := copyResources(used) + addResources(before, replacementRequests) + after := copyResources(used) + subtractResources(after, oldRequests) + addResources(after, replacementRequests) + + shortageStillPresent := false + for _, resourceName := range []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory} { + reported := resourceName == corev1.ResourceCPU && shortage.cpu || resourceName == corev1.ResourceMemory && shortage.memory + if !reported { + continue + } + if !resourceExceeds(before, node.Status.Allocatable, resourceName) { + return false + } + oldRequest := oldRequests[resourceName] + if oldRequest.Sign() <= 0 { + return false + } + shortageStillPresent = true + } + if !shortageStillPresent || !resourcesFit(after, node.Status.Allocatable) { + return false + } + if allocatablePods, ok := node.Status.Allocatable[corev1.ResourcePods]; ok && podCount > allocatablePods.Value() { + return false + } + return true +} + +func schedulerPodRequests(pod *corev1.Pod) corev1.ResourceList { + return resourcehelper.PodRequests(pod, resourcehelper.PodResourcesOptions{ + UseStatusResources: true, + InPlacePodLevelResourcesVerticalScalingEnabled: true, + }) +} + +func copyResources(resources corev1.ResourceList) corev1.ResourceList { + result := make(corev1.ResourceList, len(resources)) + for name, quantity := range resources { + result[name] = quantity.DeepCopy() + } + return result +} + +func addResources(target, values corev1.ResourceList) { + for name, value := range values { + quantity := target[name] + quantity.Add(value) + target[name] = quantity + } +} + +func subtractResources(target, values corev1.ResourceList) { + for name, value := range values { + quantity := target[name] + quantity.Sub(value) + target[name] = quantity + } +} + +func resourceExceeds(requests, allocatable corev1.ResourceList, name corev1.ResourceName) bool { + request := requests[name] + available := allocatable[name] + return request.Cmp(available) > 0 +} + +func resourcesFit(requests, allocatable corev1.ResourceList) bool { + for name, request := range requests { + if request.Sign() <= 0 { + continue + } + available, ok := allocatable[name] + if !ok || request.Cmp(available) > 0 { + return false + } + } + return true +} diff --git a/internal/controller/datadogagentinternal/resource_fallback_test.go b/internal/controller/datadogagentinternal/resource_fallback_test.go new file mode 100644 index 0000000000..f1b2b5e277 --- /dev/null +++ b/internal/controller/datadogagentinternal/resource_fallback_test.go @@ -0,0 +1,387 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// Copyright 2016-present Datadog, Inc. + +package datadogagentinternal + +import ( + "context" + "encoding/json" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConfigureResourceFallback(t *testing.T) { + tests := []struct { + name string + strategyType appsv1.DaemonSetUpdateStrategyType + maxSurge *intstr.IntOrString + budget intstr.IntOrString + enabled bool + wantMaxSurge *intstr.IntOrString + wantUnavailable *intstr.IntOrString + }{ + { + name: "surge is bounded by the existing percentage budget", + maxSurge: ptr.To(intstr.FromString("100%")), + budget: intstr.FromString("20%"), + enabled: true, + wantMaxSurge: ptr.To(intstr.FromString("20%")), + wantUnavailable: ptr.To(intstr.FromInt(0)), + }, + { + name: "absolute budget", + maxSurge: ptr.To(intstr.FromInt(20)), + budget: intstr.FromInt(2), + enabled: true, + wantMaxSurge: ptr.To(intstr.FromInt(2)), + wantUnavailable: ptr.To(intstr.FromInt(0)), + }, + { + name: "surge remains opt in", + budget: intstr.FromInt(1), + wantMaxSurge: nil, + }, + { + name: "on delete is untouched", + strategyType: appsv1.OnDeleteDaemonSetStrategyType, + maxSurge: ptr.To(intstr.FromInt(1)), + budget: intstr.FromInt(1), + wantMaxSurge: ptr.To(intstr.FromInt(1)), + }, + { + name: "zero budget disables fallback but preserves requested surge", + maxSurge: ptr.To(intstr.FromInt(3)), + budget: intstr.FromInt(0), + wantMaxSurge: ptr.To(intstr.FromInt(3)), + wantUnavailable: ptr.To(intstr.FromInt(0)), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ds := &appsv1.DaemonSet{Spec: appsv1.DaemonSetSpec{UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ + Type: tt.strategyType, + RollingUpdate: &appsv1.RollingUpdateDaemonSet{MaxSurge: tt.maxSurge}, + }}} + assert.Equal(t, tt.enabled, configureResourceFallback(ds, tt.budget)) + assert.Equal(t, tt.wantMaxSurge, ds.Spec.UpdateStrategy.RollingUpdate.MaxSurge) + assert.Equal(t, tt.wantUnavailable, ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable) + }) + } +} + +func TestResourceOnlyUnschedulable(t *testing.T) { + tests := []struct { + name string + reason string + message string + want resourceShortage + ok bool + }{ + {name: "cpu", reason: corev1.PodReasonUnschedulable, message: "0/1 nodes are available: 1 Insufficient cpu.", want: resourceShortage{cpu: true}, ok: true}, + {name: "memory and pinning affinity", reason: corev1.PodReasonUnschedulable, message: "0/3 nodes are available: 1 Insufficient memory, 2 node(s) didn't satisfy plugin(s) [NodeAffinity]. preemption: 0/3 nodes are available: 3 Preemption is not helpful for scheduling.", want: resourceShortage{memory: true}, ok: true}, + {name: "cpu and memory", reason: corev1.PodReasonUnschedulable, message: "0/1 nodes are available: 1 Insufficient cpu, 1 Insufficient memory.", want: resourceShortage{cpu: true, memory: true}, ok: true}, + {name: "taint is rejected", reason: corev1.PodReasonUnschedulable, message: "0/1 nodes are available: 1 Insufficient cpu, 1 node(s) had untolerated taint.", ok: false}, + {name: "host port is rejected", reason: corev1.PodReasonUnschedulable, message: "0/1 nodes are available: 1 Insufficient cpu, 1 node(s) didn't have free ports for the requested pod ports.", ok: false}, + {name: "ephemeral storage is rejected", reason: corev1.PodReasonUnschedulable, message: "0/1 nodes are available: 1 Insufficient ephemeral-storage.", ok: false}, + {name: "custom reason containing cpu text is rejected", reason: corev1.PodReasonUnschedulable, message: "0/1 nodes are available: 1 custom plugin: Insufficient cpu.", ok: false}, + {name: "wrong condition reason", reason: "SchedulingGated", message: "0/1 nodes are available: 1 Insufficient cpu.", ok: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pod := &corev1.Pod{Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: tt.reason, Message: tt.message}}}} + got, ok := resourceOnlyUnschedulable(pod) + assert.Equal(t, tt.ok, ok) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestTargetNodeFromDaemonSetAffinity(t *testing.T) { + pod := pendingPodForNode("node-a") + got, ok := targetNodeFromDaemonSetAffinity(pod) + require.True(t, ok) + assert.Equal(t, "node-a", got) + + ambiguous := pendingPodForNode("node-a") + ambiguous.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append( + ambiguous.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, + corev1.NodeSelectorTerm{MatchFields: []corev1.NodeSelectorRequirement{{Key: metav1.ObjectNameField, Operator: corev1.NodeSelectorOpIn, Values: []string{"node-b"}}}}, + ) + _, ok = targetNodeFromDaemonSetAffinity(ambiguous) + assert.False(t, ok) +} + +func TestResourceFitAfterOldPodRemoval(t *testing.T) { + node := &corev1.Node{Status: corev1.NodeStatus{Allocatable: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1500m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + corev1.ResourcePods: resource.MustParse("10"), + }}} + old := scheduledResourcePod("old", "old-uid", "node-a", "1", "128Mi") + replacement := scheduledResourcePod("new", "new-uid", "", "1", "128Mi") + + assert.True(t, resourceFitAfterOldPodRemoval(node, []corev1.Pod{*old}, replacement, old, resourceShortage{cpu: true})) + + tooLarge := replacement.DeepCopy() + tooLarge.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = resource.MustParse("2") + assert.False(t, resourceFitAfterOldPodRemoval(node, []corev1.Pod{*old}, tooLarge, old, resourceShortage{cpu: true}), "replacement must fit after old exits") + + noCPUOld := old.DeepCopy() + noCPUOld.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = resource.MustParse("0") + assert.False(t, resourceFitAfterOldPodRemoval(node, []corev1.Pod{*noCPUOld}, replacement, noCPUOld, resourceShortage{cpu: true}), "old Pod must contribute to the shortage") + + staleMessage := node.DeepCopy() + staleMessage.Status.Allocatable[corev1.ResourceCPU] = resource.MustParse("3") + assert.False(t, resourceFitAfterOldPodRemoval(staleMessage, []corev1.Pod{*old}, replacement, old, resourceShortage{cpu: true}), "reported shortage must still be observable") +} + +func TestSchedulerPodRequestsIncludesInitAndOverhead(t *testing.T) { + pod := scheduledResourcePod("pod", "uid", "node-a", "250m", "100Mi") + pod.Spec.InitContainers = []corev1.Container{{Name: "init", Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}}}} + pod.Spec.Overhead = corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")} + requests := schedulerPodRequests(pod) + assert.Equal(t, int64(1100), requests.Cpu().MilliValue()) +} + +func TestConsumedFallbackBudget(t *testing.T) { + now := time.Now() + ds := &appsv1.DaemonSet{Spec: appsv1.DaemonSetSpec{MinReadySeconds: 0}, Status: appsv1.DaemonSetStatus{NumberUnavailable: 1}} + old := readyPod("old", "old-uid", "node-a", "old", now.Add(-time.Minute)) + reserved := pendingPodForNode("node-a") + reserved.Name = "new" + reserved.Labels = map[string]string{appsv1.DefaultDaemonSetUniqueLabelKey: "new"} + reserved.Annotations = map[string]string{resourceFallbackOldPodAnnotation: "old-uid"} + + assert.Equal(t, 2, consumedFallbackBudget(ds, []corev1.Pod{*old, *reserved}, "new", now), "reservation is separate while old remains available") + old.DeletionTimestamp = &metav1.Time{Time: now} + assert.Equal(t, 1, consumedFallbackBudget(ds, []corev1.Pod{*old, *reserved}, "new", now), "reservation overlaps status once its node is unavailable") +} + +func TestReconcileResourceFallbackDeletesOnlyResourceBlockingOldPod(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + result, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.Equal(t, time.Second, result.RequeueAfter) + err = fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{}) + assert.True(t, apierrors.IsNotFound(err), "old Pod should be deleted") + updatedPending := &corev1.Pod{} + require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.pending), updatedPending)) + assert.Equal(t, string(fixture.old.UID), updatedPending.Annotations[resourceFallbackOldPodAnnotation]) +} + +func TestReconcileResourceFallbackKeepsOldPodDuringNodePressure(t *testing.T) { + conditions := healthyNodeConditions() + for i := range conditions { + if conditions[i].Type == corev1.NodeDiskPressure { + conditions[i].Status = corev1.ConditionTrue + } + } + fixture := newFallbackTestFixture(t, conditions) + result, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.Zero(t, result.RequeueAfter, "permanently ineligible candidates must not cause a one-second polling loop") + require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{}), "old Pod must remain during DiskPressure") + updatedPending := &corev1.Pod{} + require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.pending), updatedPending)) + assert.Empty(t, updatedPending.Annotations[resourceFallbackOldPodAnnotation], "ineligible fallback must not reserve budget") +} + +func TestReconcileResourceFallbackKeepsOldPodForHiddenSchedulerConstraints(t *testing.T) { + tests := []struct { + name string + mutate func(*corev1.Pod) + }{ + { + name: "persistent volume", + mutate: func(pod *corev1.Pod) { + pod.Spec.Volumes = []corev1.Volume{{Name: "data", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"}}}} + }, + }, + { + name: "pod affinity", + mutate: func(pod *corev1.Pod) { + pod.Spec.Affinity.PodAffinity = &corev1.PodAffinity{RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{TopologyKey: "kubernetes.io/hostname", LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "peer"}}}}} + }, + }, + { + name: "declared host port", + mutate: func(pod *corev1.Pod) { + pod.Spec.Containers[0].Ports = []corev1.ContainerPort{{ContainerPort: 8126, HostPort: 8126}} + }, + }, + { + name: "topology spread", + mutate: func(pod *corev1.Pod) { + pod.Spec.TopologySpreadConstraints = []corev1.TopologySpreadConstraint{{MaxSkew: 1, TopologyKey: "kubernetes.io/hostname", WhenUnsatisfiable: corev1.DoNotSchedule, LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "agent"}}}} + }, + }, + { + name: "custom scheduler", + mutate: func(pod *corev1.Pod) { + pod.Spec.SchedulerName = "custom-scheduler" + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + pending := &corev1.Pod{} + require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.pending), pending)) + tt.mutate(pending) + require.NoError(t, fixture.client.Update(context.Background(), pending)) + _, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{}), "old Pod must remain") + updatedPending := &corev1.Pod{} + require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.pending), updatedPending)) + assert.Empty(t, updatedPending.Annotations[resourceFallbackOldPodAnnotation]) + }) + } +} + +func TestReconcileResourceFallbackUsesLiveUnavailablePodsWhenStatusLags(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + liveDS := &appsv1.DaemonSet{} + require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.ds), liveDS)) + liveDS.Status.DesiredNumberScheduled = 2 + require.NoError(t, fixture.client.Status().Update(context.Background(), liveDS)) + + unavailable := readyPod("unavailable", "unavailable-uid", "node-b", "old-revision", time.Now().Add(-time.Minute)) + unavailable.Namespace = fixture.ds.Namespace + unavailable.Labels["app"] = "agent" + unavailable.OwnerReferences = []metav1.OwnerReference{daemonSetOwner(fixture.ds)} + unavailable.Status.Conditions[0].Status = corev1.ConditionFalse + require.NoError(t, fixture.client.Create(context.Background(), unavailable)) + + result, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.Zero(t, result.RequeueAfter, "an already-consumed budget must not cause a one-second polling loop") + require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{}), "fallback must not exceed maxUnavailable while DaemonSet status lags") +} + +func TestReconcileResourceFallbackRejectsForeignDaemonSet(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + liveDS := &appsv1.DaemonSet{} + require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.ds), liveDS)) + liveDS.OwnerReferences[0].UID = "foreign-ddai" + require.NoError(t, fixture.client.Update(context.Background(), liveDS)) + + _, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{}), "foreign DaemonSet Pods must never be deleted") +} + +type fallbackTestFixture struct { + client client.Client + reconciler *Reconciler + ddai *datadoghqv1alpha1.DatadogAgentInternal + ds *appsv1.DaemonSet + old *corev1.Pod + pending *corev1.Pod +} + +func newFallbackTestFixture(t *testing.T, nodeConditions []corev1.NodeCondition) fallbackTestFixture { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, appsv1.AddToScheme(scheme)) + require.NoError(t, datadoghqv1alpha1.AddToScheme(scheme)) + + ddai := &datadoghqv1alpha1.DatadogAgentInternal{ObjectMeta: metav1.ObjectMeta{Name: "agent", Namespace: "default", UID: "ddai-uid"}} + ds := testFallbackDaemonSet(t, ddai) + old := readyPod("old", "old-uid", "node-a", "old-revision", time.Now().Add(-time.Minute)) + pending := pendingPodForNode("node-a") + pending.ObjectMeta = metav1.ObjectMeta{Name: "new", Namespace: "default", UID: "new-uid", Labels: map[string]string{"app": "agent", appsv1.DefaultDaemonSetUniqueLabelKey: "new-revision"}, OwnerReferences: []metav1.OwnerReference{daemonSetOwner(ds)}} + pending.Spec.Containers = []corev1.Container{{Name: "agent", Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}}}} + pending.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: corev1.PodReasonUnschedulable, Message: "0/1 nodes are available: 1 Insufficient cpu."}} + old.Namespace = "default" + old.Labels["app"] = "agent" + old.OwnerReferences = []metav1.OwnerReference{daemonSetOwner(ds)} + old.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = resource.MustParse("1") + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-a"}, Status: corev1.NodeStatus{Allocatable: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1500m"), corev1.ResourceMemory: resource.MustParse("1Gi"), corev1.ResourcePods: resource.MustParse("10")}, Conditions: nodeConditions}} + revision := controllerRevisionForTemplate(t, ds, "new-revision") + + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(ddai, ds, old, pending, node, revision).WithIndex(&corev1.Pod{}, apiPodNodeNameField, func(obj client.Object) []string { + pod := obj.(*corev1.Pod) + if pod.Spec.NodeName == "" { + return nil + } + return []string{pod.Spec.NodeName} + }).Build() + r := &Reconciler{client: c, apiReader: c} + return fallbackTestFixture{client: c, reconciler: r, ddai: ddai, ds: ds, old: old, pending: pending} +} + +func healthyNodeConditions() []corev1.NodeCondition { + return []corev1.NodeCondition{ + {Type: corev1.NodeReady, Status: corev1.ConditionTrue}, + {Type: corev1.NodeMemoryPressure, Status: corev1.ConditionFalse}, + {Type: corev1.NodeDiskPressure, Status: corev1.ConditionFalse}, + {Type: corev1.NodePIDPressure, Status: corev1.ConditionFalse}, + {Type: corev1.NodeNetworkUnavailable, Status: corev1.ConditionFalse}, + } +} + +func testFallbackDaemonSet(t *testing.T, ddai *datadoghqv1alpha1.DatadogAgentInternal) *appsv1.DaemonSet { + t.Helper() + return &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: "agent", Namespace: "default", UID: "ds-uid", Generation: 2, OwnerReferences: []metav1.OwnerReference{{APIVersion: datadoghqv1alpha1.GroupVersion.String(), Kind: "DatadogAgentInternal", Name: ddai.Name, UID: ddai.UID, Controller: ptr.To(true)}}}, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "agent"}}, + Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "agent"}}, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "agent"}}}}, + UpdateStrategy: appsv1.DaemonSetUpdateStrategy{Type: appsv1.RollingUpdateDaemonSetStrategyType, RollingUpdate: &appsv1.RollingUpdateDaemonSet{MaxSurge: ptr.To(intstr.FromInt(1)), MaxUnavailable: ptr.To(intstr.FromInt(0))}}, + }, + Status: appsv1.DaemonSetStatus{ObservedGeneration: 2, DesiredNumberScheduled: 1}, + } +} + +func controllerRevisionForTemplate(t *testing.T, ds *appsv1.DaemonSet, hash string) *appsv1.ControllerRevision { + t.Helper() + templateJSON, err := json.Marshal(ds.Spec.Template) + require.NoError(t, err) + var templatePatch map[string]any + require.NoError(t, json.Unmarshal(templateJSON, &templatePatch)) + templatePatch["$patch"] = "replace" + data, err := json.Marshal(map[string]any{"spec": map[string]any{"template": templatePatch}}) + require.NoError(t, err) + return &appsv1.ControllerRevision{ObjectMeta: metav1.ObjectMeta{Name: "agent-" + hash, Namespace: ds.Namespace, UID: types.UID("revision-uid"), Labels: map[string]string{appsv1.DefaultDaemonSetUniqueLabelKey: hash}, OwnerReferences: []metav1.OwnerReference{daemonSetOwner(ds)}}, Revision: 2, Data: runtime.RawExtension{Raw: data}} +} + +func daemonSetOwner(ds *appsv1.DaemonSet) metav1.OwnerReference { + return metav1.OwnerReference{APIVersion: appsv1.SchemeGroupVersion.String(), Kind: "DaemonSet", Name: ds.Name, UID: ds.UID, Controller: ptr.To(true)} +} + +func pendingPodForNode(nodeName string) *corev1.Pod { + return &corev1.Pod{Spec: corev1.PodSpec{Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchFields: []corev1.NodeSelectorRequirement{{Key: metav1.ObjectNameField, Operator: corev1.NodeSelectorOpIn, Values: []string{nodeName}}}}}}}}}} +} + +func scheduledResourcePod(name, uid, nodeName, cpu, memory string) *corev1.Pod { + return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(uid)}, Spec: corev1.PodSpec{NodeName: nodeName, Containers: []corev1.Container{{Name: "agent", Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse(cpu), corev1.ResourceMemory: resource.MustParse(memory)}}}}}} +} + +func readyPod(name, uid, nodeName, revision string, readyAt time.Time) *corev1.Pod { + pod := scheduledResourcePod(name, uid, nodeName, "100m", "128Mi") + pod.Labels = map[string]string{appsv1.DefaultDaemonSetUniqueLabelKey: revision} + pod.Status = corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue, LastTransitionTime: metav1.NewTime(readyAt)}}} + return pod +} diff --git a/internal/controller/datadogagentinternal_controller.go b/internal/controller/datadogagentinternal_controller.go index 53429b537d..0abd66eb68 100644 --- a/internal/controller/datadogagentinternal_controller.go +++ b/internal/controller/datadogagentinternal_controller.go @@ -13,6 +13,7 @@ import ( corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -25,10 +26,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + apicommon "github.com/DataDog/datadog-operator/api/datadoghq/common" "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" "github.com/DataDog/datadog-operator/internal/controller/datadogagent/object" "github.com/DataDog/datadog-operator/internal/controller/datadogagentinternal" + "github.com/DataDog/datadog-operator/pkg/constants" "github.com/DataDog/datadog-operator/pkg/controller/utils/datadog" "github.com/DataDog/datadog-operator/pkg/kubernetes" ) @@ -46,6 +49,8 @@ type DatadogAgentInternalReconciler struct { // +kubebuilder:rbac:groups=datadoghq.com,resources=datadogagentinternals,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=datadoghq.com,resources=datadogagentinternals/status,verbs=get;update;patch // +kubebuilder:rbac:groups=datadoghq.com,resources=datadogagentinternals/finalizers,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;patch;delete +// +kubebuilder:rbac:groups=apps,resources=controllerrevisions,verbs=get;list;watch // Reconcile loop for DatadogAgent. func (r *DatadogAgentInternalReconciler) Reconcile(ctx context.Context, ddai *v1alpha1.DatadogAgentInternal) (ctrl.Result, error) { @@ -58,27 +63,33 @@ func (r *DatadogAgentInternalReconciler) Reconcile(ctx context.Context, ddai *v1 // SetupWithManager creates a new DatadogAgent controller. func (r *DatadogAgentInternalReconciler) SetupWithManager(mgr ctrl.Manager, metricForwardersMgr datadog.MetricsForwardersManager) error { + generationChanged := ctrlbuilder.WithPredicates(predicate.GenerationChangedPredicate{}) builder := ctrl.NewControllerManagedBy(mgr). - Owns(&corev1.Secret{}). - Owns(&corev1.ConfigMap{}). - Owns(&appsv1.DaemonSet{}). - Owns(&appsv1.Deployment{}). - Owns(&rbacv1.Role{}). - Owns(&rbacv1.RoleBinding{}). - Owns(&corev1.ServiceAccount{}). + Owns(&corev1.Secret{}, generationChanged). + Owns(&corev1.ConfigMap{}, generationChanged). + Owns(&appsv1.DaemonSet{}, generationChanged). + Owns(&appsv1.Deployment{}, generationChanged). + Owns(&rbacv1.Role{}, generationChanged). + Owns(&rbacv1.RoleBinding{}, generationChanged). + Owns(&corev1.ServiceAccount{}, generationChanged). // We let PlatformInfo supply PDB object based on the current API version - Owns(r.PlatformInfo.CreatePDBObject()). - Owns(&networkingv1.NetworkPolicy{}) + Owns(r.PlatformInfo.CreatePDBObject(), generationChanged). + Owns(&networkingv1.NetworkPolicy{}, generationChanged) // DatadogAgent is namespaced whereas ClusterRole and ClusterRoleBinding are // cluster-scoped. That means that DatadogAgent cannot be their owner, and // we cannot use .Owns(). handlerEnqueue := handler.EnqueueRequestsFromMapFunc(enqueueIfOwnedByDatadogAgentInternal) - builder.Watches(&rbacv1.ClusterRole{}, handlerEnqueue) - builder.Watches(&rbacv1.ClusterRoleBinding{}, handlerEnqueue) + builder.Watches(&rbacv1.ClusterRole{}, handlerEnqueue, generationChanged) + builder.Watches(&rbacv1.ClusterRoleBinding{}, handlerEnqueue, generationChanged) + builder.Watches( + &corev1.Pod{}, + handler.EnqueueRequestsFromMapFunc(enqueueDatadogAgentInternalForPod(mgr.GetAPIReader())), + ctrlbuilder.WithPredicates(resourceFallbackPodPredicate()), + ) if r.Options.ExtendedDaemonsetOptions.Enabled { - builder = builder.Owns(&edsdatadoghqv1alpha1.ExtendedDaemonSet{}) + builder = builder.Owns(&edsdatadoghqv1alpha1.ExtendedDaemonSet{}, generationChanged) } if r.Options.SupportCilium { @@ -88,7 +99,7 @@ func (r *DatadogAgentInternalReconciler) SetupWithManager(mgr ctrl.Manager, metr Version: "v2", Kind: "CiliumNetworkPolicy", }) - builder = builder.Owns(policy) + builder = builder.Owns(policy, generationChanged) } var builderOptions []ctrlbuilder.ForOption @@ -101,17 +112,81 @@ func (r *DatadogAgentInternalReconciler) SetupWithManager(mgr ctrl.Manager, metr }, })) } + builderOptions = append(builderOptions, ctrlbuilder.WithPredicates(predicate.GenerationChangedPredicate{})) or := reconcile.AsReconciler[*v1alpha1.DatadogAgentInternal](r.Client, r) - if err := builder.For(&datadoghqv1alpha1.DatadogAgentInternal{}, builderOptions...).WithEventFilter(predicate.GenerationChangedPredicate{}).Complete(or); err != nil { + if err := builder.For(&datadoghqv1alpha1.DatadogAgentInternal{}, builderOptions...).Complete(or); err != nil { return err } - r.internal = datadogagentinternal.NewReconciler(r.Options, r.Client, r.PlatformInfo, r.Scheme, r.Recorder, metricForwardersMgr) + r.internal = datadogagentinternal.NewReconciler(r.Options, r.Client, mgr.GetAPIReader(), r.PlatformInfo, r.Scheme, r.Recorder, metricForwardersMgr) return nil } +func enqueueDatadogAgentInternalForPod(reader client.Reader) handler.MapFunc { + return func(ctx context.Context, obj client.Object) []reconcile.Request { + pod, ok := obj.(*corev1.Pod) + if !ok || pod.Labels[apicommon.AgentDeploymentComponentLabelKey] != constants.DefaultAgentResourceSuffix { + return nil + } + podOwner := metav1.GetControllerOf(pod) + if podOwner == nil || podOwner.APIVersion != appsv1.SchemeGroupVersion.String() || podOwner.Kind != "DaemonSet" { + return nil + } + ds := &appsv1.DaemonSet{} + if err := reader.Get(ctx, client.ObjectKey{Namespace: pod.Namespace, Name: podOwner.Name}, ds); err != nil || ds.UID != podOwner.UID { + return nil + } + ddaiOwner := metav1.GetControllerOf(ds) + if ddaiOwner == nil || ddaiOwner.APIVersion != datadoghqv1alpha1.GroupVersion.String() || ddaiOwner.Kind != "DatadogAgentInternal" { + return nil + } + return []reconcile.Request{{NamespacedName: client.ObjectKey{Namespace: ds.Namespace, Name: ddaiOwner.Name}}} + } +} + +func resourceFallbackPodPredicate() predicate.Predicate { + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + pod, ok := e.Object.(*corev1.Pod) + return ok && resourceFallbackSchedulingCondition(pod) != nil + }, + UpdateFunc: func(e event.UpdateEvent) bool { + oldPod, oldOK := e.ObjectOld.(*corev1.Pod) + newPod, newOK := e.ObjectNew.(*corev1.Pod) + if !oldOK || !newOK { + return false + } + return resourceFallbackConditionChanged(oldPod, newPod, corev1.PodScheduled) || resourceFallbackConditionChanged(oldPod, newPod, corev1.PodReady) + }, + DeleteFunc: func(event.DeleteEvent) bool { return true }, + GenericFunc: func(event.GenericEvent) bool { return false }, + } +} + +func resourceFallbackConditionChanged(oldPod, newPod *corev1.Pod, conditionType corev1.PodConditionType) bool { + oldCondition := podCondition(oldPod, conditionType) + newCondition := podCondition(newPod, conditionType) + if oldCondition == nil || newCondition == nil { + return oldCondition != newCondition + } + return oldCondition.Status != newCondition.Status || oldCondition.Reason != newCondition.Reason || oldCondition.Message != newCondition.Message +} + +func podCondition(pod *corev1.Pod, conditionType corev1.PodConditionType) *corev1.PodCondition { + for i := range pod.Status.Conditions { + if pod.Status.Conditions[i].Type == conditionType { + return &pod.Status.Conditions[i] + } + } + return nil +} + +func resourceFallbackSchedulingCondition(pod *corev1.Pod) *corev1.PodCondition { + return podCondition(pod, corev1.PodScheduled) +} + func enqueueIfOwnedByDatadogAgentInternal(ctx context.Context, obj client.Object) []reconcile.Request { labels := obj.GetLabels() diff --git a/internal/controller/datadogagentinternal_controller_test.go b/internal/controller/datadogagentinternal_controller_test.go new file mode 100644 index 0000000000..d51af40e36 --- /dev/null +++ b/internal/controller/datadogagentinternal_controller_test.go @@ -0,0 +1,75 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// Copyright 2016-present Datadog, Inc. + +package controller + +import ( + "context" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/event" + + apicommon "github.com/DataDog/datadog-operator/api/datadoghq/common" + datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" + "github.com/DataDog/datadog-operator/pkg/constants" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResourceFallbackPodPredicate(t *testing.T) { + predicate := resourceFallbackPodPredicate() + oldPod := &corev1.Pod{Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: corev1.PodReasonUnschedulable, Message: "old"}}}} + newPod := oldPod.DeepCopy() + newPod.Status.Conditions[0].Message = "0/1 nodes are available: 1 Insufficient cpu." + assert.True(t, predicate.Update(event.UpdateEvent{ObjectOld: oldPod, ObjectNew: newPod}), "PodScheduled message-only updates must enqueue") + + readyPod := newPod.DeepCopy() + readyPod.Status.Conditions = append(readyPod.Status.Conditions, corev1.PodCondition{Type: corev1.PodReady, Status: corev1.ConditionTrue}) + assert.True(t, predicate.Update(event.UpdateEvent{ObjectOld: newPod, ObjectNew: readyPod}), "PodReady transitions must release fallback reservations promptly") +} + +func TestEnqueueDatadogAgentInternalForPodFollowsDaemonSetOwner(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, appsv1.AddToScheme(scheme)) + ds := &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{ + Name: "profile-agent", + Namespace: "default", + UID: types.UID("ds-uid"), + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: datadoghqv1alpha1.GroupVersion.String(), + Kind: "DatadogAgentInternal", + Name: "profile-ddai", + UID: types.UID("ddai-uid"), + Controller: ptr.To(true), + }}, + }} + reader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(ds).Build() + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "profile-agent-new", + Namespace: "default", + Labels: map[string]string{apicommon.AgentDeploymentComponentLabelKey: constants.DefaultAgentResourceSuffix}, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: appsv1.SchemeGroupVersion.String(), + Kind: "DaemonSet", + Name: ds.Name, + UID: ds.UID, + Controller: ptr.To(true), + }}, + }} + + requests := enqueueDatadogAgentInternalForPod(reader)(context.Background(), pod) + require.Len(t, requests, 1) + assert.Equal(t, "default", requests[0].Namespace) + assert.Equal(t, "profile-ddai", requests[0].Name) + + pod.OwnerReferences[0].UID = "wrong-uid" + assert.Empty(t, enqueueDatadogAgentInternalForPod(reader)(context.Background(), pod), "stale Pod owner UIDs must not enqueue") +} diff --git a/internal/controller/testutils/renderer/renderer.go b/internal/controller/testutils/renderer/renderer.go index 1f59f42a98..57f6b3dd83 100644 --- a/internal/controller/testutils/renderer/renderer.go +++ b/internal/controller/testutils/renderer/renderer.go @@ -200,7 +200,7 @@ func Render(opts Options) ([]client.Object, *runtime.Scheme, error) { ddaiOpts := datadogagentinternal.ReconcilerOptions{ SupportCilium: opts.SupportCilium, } - ddaiReconciler := datadogagentinternal.NewReconciler(ddaiOpts, fakeClient, platformInfo, scheme, recorder, noopForwarder{}) + ddaiReconciler := datadogagentinternal.NewReconciler(ddaiOpts, fakeClient, fakeClient, platformInfo, scheme, recorder, noopForwarder{}) for i := range ddaiList.Items { ddai := &ddaiList.Items[i] diff --git a/pkg/config/config.go b/pkg/config/config.go index 01e95e6ccb..701f940747 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -129,10 +129,11 @@ func CacheOptions(logger logr.Logger, opts WatchOptions) cache.Options { } } - if opts.DatadogAgentProfileEnabled || opts.UntaintControllerEnabled { - // For the profiles feature and untaint controller we need to list agent pods. - // The profiles feature needs node name and labels; the untaint controller also needs - // Status.Conditions to check readiness. Pods are watched in DatadogAgent namespace(s). + if opts.DatadogAgentEnabled || opts.DatadogAgentProfileEnabled || opts.UntaintControllerEnabled { + // The Agent, profiles, and untaint controllers need to watch Agent Pods. + // The profiles feature needs node name and labels. The untaint and Agent + // resource-fallback controllers need Status.Conditions for prompt readiness + // and scheduling reconciliation. Pods are watched in DatadogAgent namespace(s). agentNamespaces := GetWatchNamespacesFromEnv(logger, AgentWatchNamespaceEnvVar) logger.Info("Pod cache enabled", "watching Pods in namespaces", slices.Collect(maps.Keys(agentNamespaces))) byObject[podObj] = cache.ByObject{ @@ -148,19 +149,21 @@ func CacheOptions(logger logr.Logger, opts WatchOptions) cache.Options { newPod := &corev1.Pod{ TypeMeta: pod.TypeMeta, ObjectMeta: v1.ObjectMeta{ - Namespace: pod.Namespace, - Name: pod.Name, - Labels: pod.Labels, + Namespace: pod.Namespace, + Name: pod.Name, + UID: pod.UID, + Labels: pod.Labels, + OwnerReferences: pod.OwnerReferences, }, Spec: corev1.PodSpec{ NodeName: pod.Spec.NodeName, }, } + newPod.Status.Conditions = pod.Status.Conditions - // The untaint controller needs Pod.Status.Conditions (readiness check) - // and Pod.Status.StartTime (readiness-timeout clock). + // The untaint controller also needs Pod.Status.StartTime for its + // readiness-timeout clock. if opts.UntaintControllerEnabled { - newPod.Status.Conditions = pod.Status.Conditions newPod.Status.StartTime = pod.Status.StartTime } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 7a242440a8..47d8987611 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -118,6 +118,7 @@ func Test_CacheConfig(t *testing.T) { wantObjectConfig: map[client.Object]objectConfig{ agentObj: {configured: true, namespaces: []string{"system"}}, + podObj: {configured: true, namespaces: []string{"system"}}, csiDriverObj: {configured: true, namespaces: []string{"default"}}, csiDaemonSetObj: {configured: true, namespaces: []string{"system", "default"}}, }, @@ -178,7 +179,7 @@ func Test_CacheConfig(t *testing.T) { }, }, { - name: "Only Agent enabled; Monitor enabled without namespace config. Other CRDs, Pods, Nodes not configured", + name: "Only Agent enabled; Monitor enabled without namespace config. Agent Pods are configured; other CRDs and Nodes are not", watchOptions: WatchOptions{ DatadogAgentEnabled: true, @@ -200,13 +201,13 @@ func Test_CacheConfig(t *testing.T) { monitorObj: {configured: true, namespaces: []string{"datadog"}}, sloObj: {configured: false}, profileObj: {configured: false}, - podObj: {configured: false}, + podObj: {configured: true, namespaces: []string{"agentNs1", "agentNs2"}}, nodeObj: {configured: false}, csiDriverObj: {configured: false}, }, }, { - name: "DAP disabled, Introspection enabled; Node uses nil namespace; Pods, Profiles are not configured", + name: "DAP disabled, Introspection enabled; Node uses nil namespace; Agent Pods are configured, Profiles are not", watchOptions: WatchOptions{ DatadogAgentEnabled: true, @@ -229,7 +230,7 @@ func Test_CacheConfig(t *testing.T) { monitorObj: {configured: false}, sloObj: {configured: false}, profileObj: {configured: false}, - podObj: {configured: false}, + podObj: {configured: true, namespaces: []string{"agentNs1", "agentNs2"}}, nodeObj: {configured: true, namespaces: nil}, csiDriverObj: {configured: false}, }, From 8c9651d144b5fdc4a1a2b3dc3a40579f0e4225d3 Mon Sep 17 00:00:00 2001 From: "ali.benabdallah" Date: Wed, 22 Jul 2026 18:24:19 +0200 Subject: [PATCH 02/16] Prototype prepared Agent rollout handoffs --- docs/agent_host_network_surge_poc.md | 58 ++ docs/agent_zero_gap_rollouts_rfc.md | 163 +++ docs/agent_zero_gap_rollouts_rfc_appendix.md | 968 ++++++++++++++++++ experiments/openkruise-prepull/README.md | 378 +++++++ .../automatic-predownload.yaml | 43 + experiments/openkruise-prepull/base.yaml | 100 ++ .../imagepulljob-activation-failure.yaml | 18 + .../openkruise-prepull/imagepulljob-bad.yaml | 18 + .../openkruise-prepull/imagepulljob-good.yaml | 18 + .../controller_reconcile_agent.go | 47 +- .../datadogagentinternal/prepared_rollout.go | 447 ++++++++ .../prepared_rollout_test.go | 225 ++++ .../datadogagentinternal/resource_fallback.go | 262 ++++- .../resource_fallback_test.go | 175 ++++ .../datadogagentinternal_controller.go | 25 +- .../datadogagentinternal_controller_test.go | 6 + .../testutils/renderer/render_e2e_test.go | 125 +++ pkg/config/config.go | 3 + pkg/config/config_test.go | 35 + 19 files changed, 3087 insertions(+), 27 deletions(-) create mode 100644 docs/agent_host_network_surge_poc.md create mode 100644 docs/agent_zero_gap_rollouts_rfc.md create mode 100644 docs/agent_zero_gap_rollouts_rfc_appendix.md create mode 100644 experiments/openkruise-prepull/README.md create mode 100644 experiments/openkruise-prepull/automatic-predownload.yaml create mode 100644 experiments/openkruise-prepull/base.yaml create mode 100644 experiments/openkruise-prepull/imagepulljob-activation-failure.yaml create mode 100644 experiments/openkruise-prepull/imagepulljob-bad.yaml create mode 100644 experiments/openkruise-prepull/imagepulljob-good.yaml create mode 100644 internal/controller/datadogagentinternal/prepared_rollout.go create mode 100644 internal/controller/datadogagentinternal/prepared_rollout_test.go diff --git a/docs/agent_host_network_surge_poc.md b/docs/agent_host_network_surge_poc.md new file mode 100644 index 0000000000..16201e2000 --- /dev/null +++ b/docs/agent_host_network_surge_poc.md @@ -0,0 +1,58 @@ +# Prepared host-network Agent surge PoC + +See [RFC: Prepared per-node Agent rollouts](agent_zero_gap_rollouts_rfc.md) for +the lifecycle proposal, trade-offs, alternatives, and validation plan. + +This PoC keeps `override.nodeAgent.hostNetwork: true` while allowing a native +DaemonSet surge Pod to be scheduled beside the old Agent Pod. + +It is explicitly enabled with: + +```yaml +metadata: + annotations: + experimental.agent.datadoghq.com/host-network-surge-prepared: "true" +spec: + override: + nodeAgent: + hostNetwork: true + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 +``` + +The Operator first performs an `arm` rollout with `maxSurge: 0`. Once that +exact revision is fully Available it emits a `standby` template, removes +container port declarations, and changes the native DaemonSet strategy to +`maxUnavailable: 0`. Removing the declarations is necessary because Kubernetes +defaults every declared `containerPort` to the same `hostPort` for a +host-network Pod. The processes can still bind the node ports without PodSpec +port declarations. + +For DatadogAgentProfiles, the PoC narrows the standard Pod anti-affinity only +enough to let old and new revisions of the same DDA and profile overlap. Other +profiles and other DDA installations remain excluded. + +## Prepared-mode contract + +The first pilot fails closed unless the rendered Pod has exactly: + +- optimized Linux `agent` and `trace-agent` containers; +- standard `init-volume` and `init-config` init containers; +- `hostNetwork: true` and a RollingUpdate strategy; and +- no custom lifecycle hooks, reserved rollout paths, unsupported anti-affinity, + or custom commands. + +The Operator injects per-component node lock paths and Pod-private state paths, +bypasses `trace-loader`, and replaces network probes with state-file exec +probes. Startup/liveness accept Prepared or Active; readiness accepts only +Active. Once both containers have `Started=true`, the Operator annotates the +replacement with the old Pod UID and deletes that exact UID within the +`maxUnavailable` budget. The node locks keep the new processes asleep until the +old processes finish stopping. + +Emissary and all additional Agent containers must be disabled for this pilot. +CPU/memory fallback is independent and remains off unless +`experimental.agent.datadoghq.com/resource-fallback: "true"` is also set. diff --git a/docs/agent_zero_gap_rollouts_rfc.md b/docs/agent_zero_gap_rollouts_rfc.md new file mode 100644 index 0000000000..d487003085 --- /dev/null +++ b/docs/agent_zero_gap_rollouts_rfc.md @@ -0,0 +1,163 @@ +# RFC: Prepared per-node Agent rollouts + +- Status: Draft +- Last updated: 2026-07-22 +- Owners: Agent and Datadog Operator + +## Decision + +Prototype native DaemonSet surge with an Agent **Prepared** state and an +Operator-controlled, one-node-at-a-time handoff. + +The replacement Pod is scheduled, its images and init containers complete, and +the real Agent processes start before the old Pod is terminated. Prepared +processes do not bind shared ports or UDS paths, start active collectors, mutate +shared log state, or acquire exclusive kernel resources. A failed pull, init, or +preparation therefore leaves the old Agent running unless the user-enabled +resource fallback has already deleted it. + +This phase removes the dominant pull/init/process-start delay. It does not claim +strict zero-gap handoff: releasing an old listener or collector and activating +its replacement leaves a smaller residual interval. If strict endpoint +continuity is required, the preferred extension is a stable node-local endpoint +holder or socket file-descriptor handoff. + +## Why this is needed + +The current DaemonSet rollout deletes the old Agent before scheduling, pulling, +initializing, and starting its replacement. A slow or failed pull can leave a +node without an Agent indefinitely. A slow system-probe teardown can also keep +the whole old Pod Terminating after sibling containers exit. + +Native `maxSurge` gives us create-before-delete, but production Agent Pods have +three overlap constraints: + +- With `hostNetwork: true`, Kubernetes defaults every declared `containerPort` + to the same `hostPort`; the scheduler rejects the second Pod. +- Both Pods mount the same UDS and host paths. A second active process can bind + or unlink the socket, duplicate log/check collection, or contend for kernel + resources. +- The scheduler accounts both Pods' CPU and memory requests. + +The design keeps `hostNetwork: true`. The Operator removes all regular and init +container `ports:` declarations from an explicitly compatible template; Agent +Prepared mode prevents runtime binds until activation. Shared hostPath mounting +itself is not the UDS conflict—the bind/unlink behavior is. + +## Proposed lifecycle + +1. Enabling the experiment first renders an **arm** revision with `maxSurge: 0`. + It installs the lock-aware Agent, state files, exec probes, and narrowed + profile anti-affinity through a conventional rollout. This bootstrap is + required because a legacy Agent does not hold the ownership lock. +2. Once the exact arm template is fully updated, Ready, and Available, the + Operator renders a **standby** revision. It keeps `hostNetwork: true`, strips + PodSpec port declarations, sets `maxUnavailable: 0`, and derives `maxSurge` + from the user's existing `maxUnavailable` budget. +3. A replacement constructs the real process graph, writes `prepared` to its + Pod-private state file, then waits on a per-component advisory `flock` in a + stable node hostPath. Startup and liveness exec probes accept Prepared; + readiness accepts only Active. +4. When every supported replacement container is Running and + `ContainerStatus.Started=true`, the Operator persists a token on that Pod and + UID-precondition deletes the old Pod. The old processes release their locks + only after stopping. +5. The replacements acquire the locks, start listeners and collectors, write + `active`, and become Ready. The token remains charged until that happens. + +The handoff gate is necessary because native `maxSurge` bounds preparation, not +termination and activation. Kubernetes frees a surge slot as soon as old-Pod +deletion begins. Without Active acknowledgement, many nodes could enter a long +handoff concurrently even with a small `maxSurge`. + +Prepared health must not use HTTP, TCP, or gRPC probes. Both host-network Pods use the +node IP, so a replacement probe can accidentally hit the old Agent and trigger +premature deletion. Exec probes must identify the process inside the container. + +The first cluster experiment deliberately supports only the optimized Linux +core and trace containers plus the standard `init-volume` and `init-config` +init containers. It bypasses `trace-loader`, which otherwise binds APM +endpoints before the trace process can wait. Emissary, process/system/security +agents, OTel, host profiler, and other sidecars are rejected or disabled until +their pre-Prepared side effects are audited. + +## Capacity policy + +The correct default is honest double requests during overlap. Assigning Agent +requests to a different "port-holder" Pod is not request transfer: scheduling, +QoS, CPU shares, memory protection, and eviction accounting apply to the holder's +cgroup, leaving the Agent under-requested. + +For constrained nodes, retain the PoC's resource-fit classification behind the +separate `experimental.agent.datadoghq.com/resource-fallback` opt-in. The +Operator may delete an old Pod only when the +replacement is Pending solely for CPU or memory, the node is healthy, supported +scheduling constraints are revalidated, removing that exact old Pod appears to +make the replacement fit, and the Operator first persists a handoff token. + +Normal approved-but-not-Active handoffs and fallback reserved/deleted-but-not- +Active nodes share one ledger and must not exceed the configured +`maxUnavailable`. A fallback token is reserved before the UID-preconditioned +delete and released only after the replacement reports Active. + +This fallback is best effort, not proof. Another Pod can take the freed capacity +before the Agent schedules, leaving no old Agent and a still-Pending replacement. +The replacement can also encounter a later pull or init failure after the old +Agent is gone. +Clusters requiring deterministic headroom can run a separate low-priority +Agent-sized placeholder on each node, at the cost of reserving overlap capacity +continuously. + +## Alternatives and trade-offs + +| Option | Benefit | Main limitation | Position | +|---|---|---|---| +| Native surge + Prepared Agent + handoff gate | Normal path preserves old through pull/init/start; no permanent data-plane hop | Agent/Operator lifecycle work, temporary double requests, residual activation gap | Lead prototype | +| Stable endpoint-holder DaemonSet | Preserves public ports and UDS inode; can drain/buffer | Tier-0 proxy, protocol/origin fidelity, its own upgrade problem; does not fence logs/system-probe | Prototype if continuity is required | +| Holder also owns Agent requests | Appears to avoid double requests | Requests protect the wrong Pod/cgroup | Reject | +| Per-node request placeholder | Honest deterministic surge headroom | Permanently reserves a second Agent-sized slot; global priority can consume it | Optional capacity policy | +| OpenKruise Standard surge | Partition, pause, node selection, PreDelete hooks | Same hostPort, UDS, request, and handoff-budget problems as native surge | Focused comparison | +| OpenKruise `InPlaceIfPossible` | One Pod; preserves unaffected containers and avoids duplicate requests | Changed container stops before replacement starts; failed pull can leave it down; unsupported changes recreate Pod | Complementary optimization | +| CSI | Can provide per-Pod/shared mount paths | Does not own or transfer a live UDS socket, bind ports, or select an active Agent | Not a solution alone | +| Service/CNI/eBPF indirection | Can move TCP/UDP endpoints off host networking | Changes UDP source/origin semantics and does not solve UDS or collectors | Environment-specific | + +OpenKruise should be evaluated as two separate experiments. Standard surge does +not remove the need for Prepared mode, and PreDelete alone does not hold a surge +slot through activation. In-place update is valuable for image-only or +single-container changes, but its image pre-download currently optimizes rather +than gates rollout, and Advanced DaemonSet has no fail-closed `InPlaceOnly` mode. +Single, non-comparable local v1.9.1 runs observed 7.845-second uncached and +4.511-second pre-pulled request gaps; a failed automatic pre-download still let +the selected Pod enter `ImagePullBackOff`. + +## Current PoC and required work + +The coordinated PoC now has three pieces under development: Agent process +locking/state, Operator arm-to-standby rendering and prepared handoff, and a +single-cluster ops override. The Operator fails closed on unsupported +containers, init containers, commands, lifecycle hooks, operating systems, +anti-affinity, and reserved volume paths. Ordinary native surge is unchanged +when the prepared-rollout annotation is absent, and resource fallback is +separately disabled by default. + +Before choosing a public API, validate the two-container pilot with numbered +metrics and traces, failed/slow pulls, failed preparation and activation, +Operator restart, and resource pressure with fallback both off and on. Expand +the allowlist only after logs/process/system-probe side effects are gated; the +full validation still includes a real two-minute system-probe teardown. Then +repeat the leading result on Linux KindVM and an experimental cluster. + +## Open decisions + +- Exact Prepared boundary and ownership groups for each Agent component. +- Authenticated transport for Prepared, handoff-approved, and Active status. +- Whether the residual activation interval is acceptable or requires a stable + endpoint holder. +- Whether constrained clusters prefer indefinite stall, explicit best-effort + fallback, or permanently reserved placeholder capacity. + +Detailed failure behavior, protocol notes, alternative analysis, validation +matrix, and primary sources are in the +[investigation appendix](agent_zero_gap_rollouts_rfc_appendix.md). The current +implementation notes are in the +[prepared host-network surge PoC](agent_host_network_surge_poc.md). diff --git a/docs/agent_zero_gap_rollouts_rfc_appendix.md b/docs/agent_zero_gap_rollouts_rfc_appendix.md new file mode 100644 index 0000000000..fc8ac3973b --- /dev/null +++ b/docs/agent_zero_gap_rollouts_rfc_appendix.md @@ -0,0 +1,968 @@ +# Appendix: Prepared per-node Agent rollout investigation + +This appendix contains the detailed constraints, protocol proposal, alternative +analysis, failure modes, validation plan, and source material for +[RFC: Prepared per-node Agent rollouts](agent_zero_gap_rollouts_rfc.md). + +- Status: Draft +- Last updated: 2026-07-22 +- Owners: Agent and Datadog Operator +- Scope: Linux Kubernetes node Agent + +## Summary + +The Agent should normally prepare its replacement on each node before the old +Agent is terminated. A failed or slow image pull, initialization, or process +startup must leave the old Agent running unless the explicitly enabled resource +fallback has already deleted it. We must retain `hostNetwork: true` where it is +configured and support the host ports, Unix domain sockets (UDS), host paths, +log state, and kernel resources used by production installations. + +The leading design combines native DaemonSet surge with an Agent prepared mode: + +1. The Operator renders `maxSurge` and `maxUnavailable: 0`. +2. Kubernetes schedules the replacement beside the old Pod and completes image + pulls and init containers. +3. The new Agent processes start in a prepared state. They validate as much as + possible but do not bind shared ports, bind or unlink shared UDS paths, tail + logs, run checks, or acquire exclusive kernel resources. +4. Prepared processes report process health through container-local exec probes. + An Operator-controlled readiness gate admits only a bounded set of prepared + Pods to become Ready, allowing the DaemonSet controller to terminate those old + Pods. +5. Each old component releases a node-local ownership lock only after it has + stopped using its shared resources. Its prepared replacement acquires that + lock and activates. +6. The replacement reports Active. Only then may the Operator admit another + prepared replacement into the handoff budget. + +This design intentionally separates Kubernetes scheduling compatibility from +runtime ownership. Removing PodSpec port declarations lets two host-network Pods +be scheduled on one node; it does not let two processes bind the same address. +The prepared state is what prevents the runtime collision. + +The surge Pod normally needs a second full set of CPU and memory requests. When +that capacity is unavailable, an optional Operator fallback can classify the new +Pod as blocked only by node CPU or memory and estimate that deleting the old Pod +would make it fit. It then deletes the old Pod within the existing +`maxUnavailable` budget. This preserves rollout progress, but explicitly falls +back to the current availability behavior on those nodes and does not reserve +the freed capacity. + +This RFC recommends continuing the prepared native-surge prototype. A stable +per-node endpoint holder is a credible alternative if preserving UDS inodes or +long-lived connections is required. A different Pod must not hold the Agent's +resource requests: Kubernetes scheduling, QoS, and cgroup protection attach +those requests to the holder, not to the Agent. OpenKruise should be evaluated +both for Standard surge controls and, separately, for in-place updates, but +neither removes the need for Agent lifecycle work. + +## Status of the current PoC + +Three coordinated experimental branches implement the first testable slice. +The Agent branch adds a pre-start `flock` gate and atomic +`prepared`/`activating`/`active`/`stopped` state for core and trace, with +additional wiring under test for process and system-probe. The health listener +is moved out of graph construction so Prepared does not bind port 5555. + +The Operator branch: + +- uses `experimental.agent.datadoghq.com/host-network-surge-prepared=true` as + the explicit opt-in and otherwise leaves native surge unchanged; +- performs a conventional `arm` rollout before emitting a `standby` surge + revision, recording phase on the PodTemplate for restart safety; +- accepts only optimized Linux `agent` + `trace-agent` and the standard + `init-volume` + `init-config` init containers in the first pilot; +- keeps `hostNetwork: true`, narrows known profile anti-affinity while arming, + bypasses `trace-loader`, and strips port declarations only in standby; +- replaces every regular-container startup, liveness, and readiness probe with + an exec check of that container's private state file; +- treats `ContainerStatus.Started=true` as proof of Prepared, persists a token + on the replacement, and UID-precondition deletes the old Pod within the + existing `maxUnavailable` budget; and +- keeps CPU/memory resource fallback behind the separate, default-off + `experimental.agent.datadoghq.com/resource-fallback=true` annotation. + +The ops branch reserves one small experimental cluster, disables Emissary and +all containers outside the first allowlist, and caps the budget at one. Unit, +render, cache-transform, and controller tests cover the phase transition, +fail-closed rendering, local probes, token reservation, and old-UID deletion. +Real image builds and cluster data-plane validation remain outstanding; none of +these annotations are a supported production API. + +## Goals + +- Start replacement containers and Agent processes before terminating their old + counterparts. +- Leave the old Agent running indefinitely when the replacement image cannot be + pulled or the replacement cannot reach the prepared state. +- Preserve `hostNetwork: true` and existing node-facing port numbers. +- Support APM, DogStatsD, OTLP, UDS, logs, checks, system-probe, and other + enabled Agent containers without two active collectors on one node. +- Bound both preparation and post-delete activation concurrency with the + existing `maxUnavailable` policy rather than introducing a fixed percentage. +- Make capacity fallback explicit, conservative, observable, and optional. +- Support image, configuration, and resource changes. +- Fail closed when the Operator or Agent cannot prove that overlap is safe. + +## Non-goals + +- Image pre-pulling as the availability mechanism. It improves latency but does + not protect against pull, initialization, or process failures. +- Running two active Agents on one node. +- Eliminating the final release/acquire/bind activation interval in this phase. + Strict endpoint continuity requires a stable holder, socket activation, or + file-descriptor handoff. +- Hiding real CPU or memory use from the scheduler. +- Replacing `hostNetwork` or host-facing ingestion endpoints as a prerequisite. +- Designing the final public Operator API before the lifecycle works in a real + cluster. +- Guaranteeing gap-free handoff on a node where the configured policy permits + the resource fallback to delete the old Agent. + +## Terminology and invariants + +An Agent component is in one of these states: + +- **Starting**: the container or process has not completed safe initialization. +- **Prepared**: the process is running and healthy but owns no active node-wide + resources. +- **Active**: the process owns its required listeners, sockets, collectors, log + state, or kernel resources and performs its normal work. +- **Draining**: the process is terminating while it still owns some resources. + +The hard invariant is that at most one instance owns each node-wide resource +group. The phase-one availability target is to remove image pull, init, and +process startup from the downtime window. Lock release followed by acquisition +and endpoint binding still has a residual interval with no Active owner. The +interval must be measured and bounded operationally; strict zero-gap continuity +requires an endpoint holder, socket activation, or file-descriptor transfer. + +Prepared is not the same as Active. Kubernetes Pod Ready must temporarily mean +"safe for the old revision to terminate" for a surged replacement. Metrics and +status must expose Prepared and Active separately so users do not mistake two +Ready Pods for two active Agents. + +The resource fallback is a larger explicit exception. With fallback disabled, +an unschedulable replacement leaves the old Agent active and the rollout +stalled. With fallback enabled, availability on selected nodes may match the +current delete-first rollout while the number of handoffs remains within the +configured budget. The current PoC does not have a separate fallback opt-in and +must gain one before production evaluation. + +## Baseline behavior + +The current delete-first lifecycle is: + +1. Kubernetes marks the old Pod for deletion. +2. Its containers terminate. A slow system-probe teardown can keep the whole + Pod Terminating after sibling Agent containers have exited. +3. Kubernetes creates and schedules the replacement. +4. The node pulls images. +5. Init containers run and the Agent processes start. + +The telemetry gap contains scheduling, image pulling, initialization, and +process startup. A bad image can extend it indefinitely. Pre-pulling only +removes part of step 4 in the successful case. + +Native DaemonSet surge reverses the destructive part of this order. With +`maxSurge > 0`, Kubernetes creates a new Pod on a node that still has an old, +available Pod. It marks the old Pod for deletion only after the new Pod is Ready +for `minReadySeconds`. A Pending, image-pull-failing, or unready replacement +therefore leaves the old Pod running. + +Native surge does not bound the complete handoff. Once an old Pod has a deletion +timestamp, the DaemonSet controller stops counting that old/new pair against +`maxSurge`, even if the old containers are still draining and the replacement +has not become Active. A merely Prepared-and-Ready replacement can therefore +release a slot and let the controller advance across the cluster. An external +Active acknowledgement and admission gate are required to keep post-delete +handoffs within budget. + +If an old Pod becomes unavailable, the DaemonSet controller may create its +replacement without charging that node to the normal surge limit. Capacity and +observability must therefore tolerate more overlap than the healthy-rollout +`maxSurge` value during simultaneous failures. + +## Constraints + +### Host networking and ports + +`hostNetwork: true` puts both Pods in the same node network namespace. In +addition, Kubernetes defaults every declared `containerPort` to the same +`hostPort` for a host-network Pod. The scheduler's host-port filter then rejects +the second Pod before either process starts. + +Disabling only Datadog's `hostPortConfig` is insufficient because other +container port declarations may remain. The prepared template must omit the +entire `ports:` list from every regular and init container. PodSpec port entries +are metadata and scheduling declarations; a process can still bind the numeric +node port at runtime. + +Omitting the declarations solves only scheduling. If the prepared process calls +`bind(2)` on the production address while the old process is listening, one of +the processes still fails. The replacement must remain non-listening until it +owns the corresponding resource group. + +Network probes are also unsafe during overlap. Both host-network Pods have the +same Pod IP, so an HTTP or TCP probe directed at the replacement's numeric port +can reach the old Agent and falsely report the replacement healthy. Prepared +mode must use exec probes or another container-private health channel. Named +port references cannot survive removal of the `ports:` declarations and must +fail validation or be rewritten. + +Services and NetworkPolicies using numeric ports continue to describe the same +runtime ports. Any external object using a named target port must be detected +where possible and documented as incompatible. The Operator cannot discover +every externally managed object from a DaemonSet template. + +### Shared UDS paths + +Mounting the same hostPath into two Pods is allowed. The conflict is caused by +two processes binding, unlinking, or cleaning up the same socket pathname. +Prepared processes must not touch the production UDS path. + +A Unix socket connection refers to a kernel socket object, commonly described +by the filesystem device and inode associated with its bound pathname. If one +process unlinks the path and another binds the same path, the new pathname names +a new socket object. Existing connected clients remain attached to the old +object until it closes; new clients resolve the new object. Stream clients will +normally observe EOF or reset when the old server exits and must reconnect. +Connected datagram behavior and retry policy must be measured for supported +clients. + +Prepared surge has the same eventual rebind as the baseline, but it avoids +rebinding while the old server is live. It is therefore no worse than the +baseline for inode continuity, but it does not preserve the inode across the +handoff. A stable endpoint holder can preserve it. + +### Active collectors and host resources + +Ports and UDS are only the scheduler-visible conflicts. A second active Agent +can also duplicate checks, tail the same logs, race on registry or offset state, +attach duplicate eBPF programs, or contend for system-probe and security +resources. Each component needs an explicit prepared boundary; a generic +"ports are free" test is not enough. + +The ownership boundary should be per component or per tightly coupled resource +group. A Pod-wide gate would keep every replacement component asleep until the +slowest old container exits. With per-component gates, a new trace-agent can +activate after the old trace-agent exits while a new system-probe continues to +wait through a long old system-probe teardown. + +### Resource requests + +The scheduler sums the requests of the old and replacement Pods during surge. +The Kubernetes DaemonSet API explicitly warns that per-node DaemonSet resource +consumption can double. This is correct accounting: both Pods exist and both +can consume memory and CPU during preparation or a fault. + +Limits do not solve scheduling because scheduling is based on requests. Lowering +only the prepared Pod's request would under-account its real use and weaken its +QoS and eviction protection. Kubernetes does not provide a general primitive +that lends a CPU or memory request from one Pod to another. + +## Proposed design + +```mermaid +sequenceDiagram + participant DS as DaemonSet controller + participant New as Replacement Agent + participant Op as Operator handoff gate + participant Old as Old Agent + DS->>New: Create surge Pod + New->>New: Pull, init, start, reach Prepared + New-->>Op: Prepared acknowledgement + Op-->>New: Approve readiness when budget permits + New-->>DS: Pod Ready + DS->>Old: Delete old Pod + Old-->>New: Release component ownership locks + New->>New: Bind and activate components + New-->>Op: Active acknowledgements + Op->>Op: Release handoff token +``` + +### 1. Explicit capability gate + +Prepared surge remains opt-in until every enabled regular and init container +supports the protocol. The Operator must validate both the requested strategy +and image capabilities before it removes port declarations or relaxes +anti-affinity. + +A safe bootstrap is a one-time conventional rollout to an Agent version that +participates in ownership locks even when prepared surge is disabled. Prepared +surge can then be enabled for the next update. Custom images need an explicit +capability declaration or a runtime handshake; version-string guesses are not +sufficient. + +If validation fails, the Operator leaves the ordinary template unchanged, +reports a status condition and warning event, and does not claim surge safety. + +Every production init container must be enumerated and audited because it runs +while the old Agent is Active. An init container may prepare private files, but +it must not mutate shared UDS paths, shared log state, host permissions, kernel +state, or other node-wide resources. Exclusive work must move into the +post-ownership activation phase. Unknown or user-supplied init containers fail +closed unless they explicitly declare a reviewed overlap capability. + +### 2. Render a schedulable surge template + +For an eligible native DaemonSet, the Operator: + +- keeps `hostNetwork: true`; +- removes all container and init-container `ports:` declarations; +- rewrites every supported HTTP or TCP startup, readiness, liveness, and + lifecycle check as a container-local exec operation; +- rejects every network probe or hook it cannot rewrite, whether its port is + numeric or named; +- permits only the known DatadogAgentProfile anti-affinity transformation; +- emits `maxUnavailable: 0`; and +- uses the user's existing `maxUnavailable` value as both `maxSurge` and the + Operator handoff budget. The resource fallback uses the same ceiling only when + separately enabled. + +No constant rollout percentage is introduced. A value such as `1` stays `1`; a +percentage stays a percentage and is resolved against desired nodes using the +same rounding semantics as Kubernetes. + +### 3. Start Agent processes in prepared mode + +Image pulling and init containers finish before regular containers start. Each +enabled Agent binary then starts in a prepared mode that performs only safe +initialization. The exact boundary must be defined per component, but the +minimum contract is: + +- parse configuration and secrets; +- initialize internal state that is private to the container or revision; +- verify executable dependencies and permissions where this has no side + effects; +- expose process health through an exec-readable file or private mechanism; +- do not bind production TCP or UDP ports; +- do not bind, unlink, chmod, or clean up production UDS paths; +- do not start checks, log tailers, network collectors, or telemetry emission; +- do not acquire system-probe, eBPF, security, or other exclusive host + resources; and +- do not mutate shared log offset or registry state. + +This state should run the real Agent binary rather than a shell sleeping before +`exec`. The objective is to pay image, container, binary startup, configuration +parsing, and safe initialization costs before the old process exits. + +### 4. Report handoff readiness without network probes + +Every supported regular container gets a startup exec probe that accepts +`prepared`, `activating`, or `active`. Kubelet then records +`ContainerStatus.Started=true` for that exact container; a restart resets it. +Liveness accepts the same states so waiting is indefinite. Readiness accepts +only `active`, so a standby replacement is never a Service endpoint and never +looks healthy merely because the old host-network listener answers. + +The Operator maintains a handoff budget derived from the user's existing +`maxUnavailable` policy. When all expected containers are Running+Started, all +expected init containers exited zero, and the old Pod remains Available on the +same node, it annotates the replacement with the old UID. That persisted +reservation charges the budget across reconcile retries and Operator restarts. +After revalidation, the Operator deletes that exact UID. No Pod readiness gate +or status write is required. + +The token is charged until the replacement reaches Active/Ready. If activation +fails after deletion, no additional node is handed off once the budget is +exhausted. Native `maxSurge` may prepare another replacement, but the Operator +does not delete its old Pod. + +### 5. Fence activation with node-local ownership locks + +Each active component holds an exclusive kernel-backed lock on a stable file in +a shared hostPath. The lock file is never deleted or replaced. The component +itself, or a helper whose lifetime is coupled to the owned resources, holds the +file descriptor until listeners and other shared resources are released. + +The prepared replacement blocks on the same lock. Once it acquires ownership it +binds production endpoints, opens shared state, initializes host collectors, and +transitions to Active. A process crash releases the advisory lock with its file +descriptor. A plain marker file is insufficient because it can be left stale. + +Locks should be split where independent activation is safe, for example: + +- core Agent and DogStatsD endpoints; +- trace-agent endpoints; +- process-agent collectors; +- logs and their shared state; +- system-probe and its kernel resources; and +- security-agent resources. + +The exact grouping is an Agent design task. Lock acquisition alone is not a +license to unlink another process's UDS path: activation must first verify that +the path is absent or belongs to a dead owner and fail safely otherwise. + +### 6. Let the Operator initiate bounded termination + +A standby Pod cannot become Ready while the old process holds its lock, so +native DaemonSet readiness ordering alone would deadlock. The Operator uses the +Started statuses described above to reserve a handoff token and delete the old +Pod. The native DaemonSet controller still creates and node-targets the surge +replacement; the Operator owns only the prepared-to-termination edge. + +New components activate as their old counterparts release ownership. Their +readiness exec probes acknowledge Active after listeners, collectors, and +shared state are operational. An activation failure therefore keeps the +replacement NotReady and its token charged instead of allowing the rollout to +sweep the cluster. + +If the replacement never reaches Prepared because of an image, init, config, or +process problem, Kubernetes never deletes the old available Pod. The rollout +stalls, which is the required safe failure mode. + +### 7. Fall back only after a conservative resource-shortage estimate + +A surged replacement may remain Pending because the node cannot fit both Pods' +requests. If resource fallback is enabled, the Operator may delete the old Pod +only after all of these checks pass: + +- the pending Pod is the current DaemonSet revision and targets exactly one + node; +- the old Pod on that node is available and belongs to the previous revision; +- the scheduler reports only insufficient CPU and/or memory plus the expected + DaemonSet node-affinity mismatch on other nodes; +- there is no nomination or deletion already in progress; +- the node is Ready, schedulable, and free of memory, disk, PID, and network + pressure; +- the Pod uses a supported scheduler, volume, affinity, topology, host-port, and + resource shape; +- both directions of required Pod anti-affinity are satisfied; +- recomputing scheduler requests shows the replacement does not fit before, + but would fit in the observed snapshot after removing the exact old Pod; and +- a persistent token plus live-state recheck keeps normal approved-but-not- + Active handoffs and fallback reserved/deleted-but-not-Active nodes within one + unified configured budget. + +The fallback token is persisted before the UID-preconditioned delete and is +released only after the replacement reports Active. + +The delete uses a UID precondition. Unknown Pod-declared constraints and +unrecognized scheduler reasons fail closed. Cluster-specific plugins installed +under the default scheduler name are not discoverable through the Pod API; the +feature therefore also needs a scheduler configuration allowlist or an explicit +operational compatibility requirement. A warning event identifies the node, old +Pod, and replacement. + +This is not an atomic capacity reservation. After the checks and old-Pod delete, +another workload or nominated Pod can consume the freed capacity before the +replacement binds. The Agent may then remain Pending with no old Agent, possibly +indefinitely. It may also encounter an image pull or init failure only after the +old Pod has been deleted. Priority discipline, a preemptible placeholder, a scheduler +reservation, or direct binding would be required to close that race. Until one +is selected, fallback is a best-effort progress mechanism and must be explicitly +opted into with this failure mode visible to users. + +This fallback must never initially trigger for `ImagePullBackOff`, failed +readiness, bad configuration, host-port conflicts, disk or PID pressure, taints, +unknown affinity, or generic scheduling errors. Those failures leave the old +Agent running unless a prior resource fallback already deleted it. + +## Failure behavior + +| Failure | Expected behavior | +|---|---| +| Slow or failed image pull | Old Agent remains Active indefinitely on the normal path; after an explicit resource fallback delete, no old Agent remains. | +| Init-container failure | Old Agent remains Active indefinitely on the normal path; after an explicit resource fallback delete, no old Agent remains. | +| Agent cannot reach Prepared | Old Agent remains Active indefinitely. | +| Replacement is Prepared, old termination is slow | Prepared components wait; each activates only after its old counterpart releases ownership. | +| Normal ownership handoff | Pull, init, and process startup are already complete, but release/acquire/bind leaves a measured residual interval with no Active owner. | +| Activation fails after old exit | That node is unavailable; its handoff token remains consumed so additional nodes are not admitted. | +| Node lacks overlap CPU or memory, fallback disabled | Old Agent remains Active and rollout stalls. | +| Node lacks overlap CPU or memory, fallback enabled | Operator may delete the old Pod after a conservative fit estimate and within budget; another workload can still win the freed capacity and extend downtime. | +| Runtime port bind fails after ownership | Component remains unhealthy, does not unlink an unknown UDS, and surfaces an activation error. | +| Operator loses API connectivity | Existing Pods keep their current state; node-local ownership does not depend on a timely Operator reconcile. | +| Node or kubelet fails | Surge cannot guarantee node-local telemetry; ordinary Kubernetes node failure behavior applies. | + +## Trade-offs of the leading design + +### Advantages + +- Uses the upstream DaemonSet controller for create-before-delete ordering. +- Keeps the old Agent through pull, initialization, and prepared-process + failures. +- Retains `hostNetwork: true` and existing runtime port numbers. +- Does not add a permanent proxy to every telemetry path. +- Adds no cluster-wide workload controller dependency. +- Preserves honest per-Pod resource requests. +- Can activate components independently during a slow multi-container teardown. +- Handoff admission and fallback deletion counts use the policy users already + understand. + +### Costs and risks + +- Requires coordinated changes across multiple Agent binaries and a new + Operator handoff coordinator. +- Temporarily needs approximately two Pods' requests on surged nodes. +- Pod Ready has prepared semantics during handoff and needs separate Active + observability. +- Native surge alone does not bound post-delete activation; a missing or faulty + Active acknowledgement could stall or over-advance the rollout. +- Removing port declarations can break named target-port consumers the Operator + cannot discover. +- Every shared host resource must be audited; an omitted side effect can create + duplicate telemetry or host contention. +- Ownership lock bootstrap and mixed Agent versions require a deliberate + compatibility rollout. +- The resource fallback deliberately gives up zero-gap behavior on constrained + nodes and cannot reserve the capacity it predicts will be freed. +- Release/acquire/bind still leaves a residual no-owner interval during a normal + handoff. +- Socket inode continuity is no better than the baseline once the old socket is + closed and rebound. + +## Alternatives + +### A. Stable per-node endpoint holder + +A small, rarely updated DaemonSet can own the public host-network ports and UDS +paths. Agent Pods listen on generation-specific private ports or socket paths. +The holder health-checks backends, atomically selects the active generation, and +optionally drains old connections. + +This is the strongest endpoint abstraction: + +- the public UDS pathname and inode can remain stable across Agent updates; +- public ports never move between Agent processes; +- TCP and HTTP connections can be drained; +- bounded UDP or stream buffering can hide a short backend restart; and +- Agents may use pod networking or unique host-network backend ports. + +It is also a new node-wide data-plane dependency: + +- a holder failure interrupts metrics and traces even when the Agent is healthy; +- the holder has its own difficult upgrade problem because a second holder + cannot bind the public endpoints; +- DogStatsD UDP and datagram UDS forwarding must preserve packet boundaries and + sender credentials or origin detection can change; +- TCP keep-alive and gRPC connections remain pinned to an old backend until + drained or reset; +- queues require explicit bounds, backpressure, and drop telemetry; +- logs, checks, system-probe, and kernel ownership are not proxyable and still + need prepared/active fencing; and +- adoption requires a one-time coordinated migration of endpoints from the + Agent to the holder. + +The holder is worth prototyping if UDS inode continuity, connection drain, or +brief ingestion buffering becomes a hard requirement. It is not required merely +to schedule the sleeping replacement. + +### B. A holder that also owns the Agent's resource requests + +This variant gives the stable holder an Agent-sized CPU and memory request and +gives Agent Pods very small or zero requests, hoping old and new Agents can share +the holder's reservation. + +Reject this design. Kubernetes does not transfer requests across Pods. CPU +shares, memory protection, QoS class, quota attribution, eviction priority, and +capacity accounting apply to the holder's cgroup. The active Agent remains +under-requested, and two Agent Pods can consume memory simultaneously despite +only one being represented to the scheduler. Limits cap consumption but do not +repair placement or QoS semantics. + +Endpoint ownership and expendable capacity reservation are also incompatible +lifecycles. A Pod that must be deleted or preempted to release capacity cannot +simultaneously provide stable ports and UDS. + +If an endpoint holder and capacity reservation are both tested, they must be +separate workloads. + +### C. Low-priority per-node surge placeholder + +A separate low-priority DaemonSet can reserve one Agent-sized slot on every +node. Agent Pods have a higher PriorityClass. When a surge Pod needs capacity, +the scheduler preempts only the placeholder; after the old Agent exits, the +placeholder returns. + +This preserves honest Agent requests and uses native scheduling. It also +permanently withholds enough allocatable capacity for two Agents per node, which +is the same capacity cost as guaranteeing every surge will fit. It cannot create +physical memory. Kubernetes priority is global, so another higher-priority +workload can evict the placeholder and consume the intended slot. Victim grace +period also adds delay. Deterministic headroom therefore requires cluster-wide +priority discipline and a short placeholder termination grace. This is an +operational policy option for clusters willing to pay for reserved headroom, not +a general default. + +### D. Start small, then resize the replacement Pod + +A custom controller could create the Prepared replacement with small CPU and +memory requests. After the old Pod exits, it could use in-place Pod resize to +raise the new Pod to the normal requests before activation. + +This avoids permanent headroom and reduces scheduler overlap, but it is not an +atomic request transfer. Resize can remain `Deferred` or become `Infeasible`; +QoS class cannot change; only CPU and memory are supported; and static CPU or +memory-manager policies, Windows, and Kubernetes version or feature gates add +constraints. Until the resize succeeds, the prepared process is under-protected +and can consume more than the scheduler accounted. DaemonSet template drift and +rollback semantics also require a custom controller. Keep this as an experiment, +not an availability foundation. + +### E. Dynamic Resource Allocation or a custom ResourceClaim + +Dynamic Resource Allocation assigns devices or other driver-managed resources. +A custom driver could serialize a synthetic "node Agent endpoint" claim, but it +would not proxy a port, preserve a Unix socket, pass a socket file descriptor, or +reserve generic CPU and memory for another Pod. Exclusive allocation would also +block the desired Prepared Pod from being scheduled concurrently. The API and +driver footprint do not buy the lifecycle primitive this design needs. + +### F. OpenKruise Advanced DaemonSet: Standard surge + +OpenKruise Standard rolling update with `maxSurge` also creates the replacement +before deleting the old Pod and supports `minReadySeconds`, partition, node +selection, pause, and PreDelete lifecycle hooks. + +It has the same fundamental overlap constraints as native surge: + +- Kubernetes still defaults and schedules host ports; +- both Pods still share host-network and UDS namespaces; +- requests still double; +- a sleeping replacement still needs a Prepared readiness contract; and +- active collectors still need Agent fencing. + +OpenKruise has no built-in resource-unschedulable fallback. Its richer rollout +and PreDelete controls may simplify an explicit handoff, so it deserves a focused +prototype, but it is an added CRD/controller/webhook/node-daemon dependency and +is not by itself the availability solution. + +PreDelete is not itself a handoff-budget primitive. A Pod in OpenKruise's +pre-deleting state can stop consuming its surge slot before the hook completes, +just as a native deleting Pod does. Bounding activation still needs an external +coordinator, or carefully controlled pause and partition progression tied to an +Active acknowledgement. + +### G. OpenKruise Advanced DaemonSet: `InPlaceIfPossible` + +An in-place update preserves the Pod UID, node, network namespace, and mounted +volumes. It restarts changed containers while unaffected containers continue. +This avoids a second Pod, rescheduling, duplicate requests, and complete Pod +teardown. It is attractive when one Agent container changes or a long +system-probe teardown should not block an unrelated component update. + +It does not overlap old and new instances of a changed container. Kubelet stops +that container before pulling and starting its new image, so a failed image pull +can leave the component down. OpenKruise image pre-download currently improves +the common case but does not gate the rollout on successful pulls. + +Supported in-place changes are narrower than arbitrary Pod-template changes. +Unsupported changes under `InPlaceIfPossible` fall back to Pod recreation. +Advanced DaemonSet does not provide a fail-closed `InPlaceOnly` mode. CPU and +memory resize support also depends on Kubernetes feature support. + +Treat in-place update as a complementary optimization, not the primitive that +satisfies the failed-pull invariant. Useful upstream contributions would be an +Advanced DaemonSet `InPlaceOnly` policy, a real image-pre-download success gate, +and explicit prepared/activation lifecycle support. + +#### Local OpenKruise v1.9.1 result + +A Kubernetes v1.36.1 Kind experiment measured one image-only update with an +uncached image and one with a standalone, successfully completed ImagePullJob. +The uncached update had a 7.845-second success-to-success request gap, including +a kubelet-reported 2.559-second pull. A different, pre-pulled image had a +4.511-second gap and was reported already present by kubelet, but retained the +container restart and readiness gap. These single runs are not a causal timing +comparison. + +A failed standalone ImagePullJob left the old Pod, container ID, restart count, +image, and traffic unchanged because the DaemonSet was not mutated. In contrast, +successful pre-pull of an incompatible image was followed by exit 127 and +`CrashLoopBackOff` after activation. Adding an environment variable caused +`InPlaceIfPossible` to recreate the Pod under a new UID. + +OpenKruise's automatic AdvancedDaemonSet pre-download is an alpha feature gate +that defaults off. With it enabled on a two-worker test, the controller created +an owned ImagePullJob and started an in-place update without waiting: the old +container exited two seconds after the job started while that job was still +active, and the Pod entered `ImagePullBackOff`. Exact commands, identities, and +timestamps are in `experiments/openkruise-prepull/README.md`. + +### H. Endpoint holder sidecar plus OpenKruise in-place update + +An endpoint holder in the same Pod can remain running while OpenKruise updates +only Agent containers. With Pod-level resources, the holder and Agent can share +one correctly scoped Pod budget, and the holder can preserve listeners during +eligible image-only updates. + +This hybrid avoids the cross-Pod request problem but does not preserve endpoints +when an unsupported change recreates the Pod. It also still has no overlapping +old and new Agent process, so the holder needs buffering to mask process startup +and cannot cover logs or kernel collectors. It is a promising optimization for +specific update classes, not a universal rollout model. + +### I. CSI for the shared UDS + +A CSI driver can provision or mount a shared directory, choose per-Pod backing +paths, and provide mount lifecycle hooks. It cannot preserve or transfer a live +socket object, bind host-network ports, select an active Agent backend, or stop a +process from unlinking the shared pathname. + +A CSI node plugin could itself own and proxy the public UDS, but then it is the +stable endpoint-holder design packaged as storage infrastructure. CSI alone does +not solve the socket ownership problem and is unnecessary for two Pods to mount +the existing hostPath. + +### J. Node-local Service, CNI, or eBPF endpoint indirection + +Agents can use pod networking and a node-local Service, NodePort, CNI redirect, +or eBPF program to expose stable node endpoints. This removes Pod host-port +reservations and can select an active backend. + +The approach changes networking and attribution semantics. UDP source identity, +DogStatsD origin detection, host reachability, NetworkPolicy behavior, and +support across customer CNIs must be validated. It does not solve UDS, logs, or +kernel ownership. It may be appropriate for a controlled internal environment +but is a larger compatibility change than prepared host-network surge. + +### K. `SO_REUSEPORT` + +Linux `SO_REUSEPORT` can allow multiple processes to bind the same TCP or UDP +address after scheduler reservations are removed. The kernel distributes flows +or datagrams between listeners; it does not provide the required active/passive +ownership. A privileged eBPF reuseport selector could implement selection, but +that is another endpoint-indirection data plane, is Linux-specific, and does +nothing for filesystem UDS paths, logs, or kernel collectors. It is not a +general handoff mechanism. + +### L. Two DaemonSets or a custom per-node rollout controller + +The Operator can manage old and new DaemonSets, or a new controller can create +one replacement Pod per selected node and coordinate explicit handoffs. +This offers full state-machine control, including scheduling timeouts and +resource fallback. + +It recreates substantial logic already present in the native DaemonSet +controller, increases API objects and reconciliation state, and still requires +the same prepared Agent behavior for ports, UDS, and active collectors. It is +justified only if native readiness-driven ordering cannot express the required +handoff. + +### M. Image pre-pull only + +An ImagePullJob, pre-pull DaemonSet, or runtime cache warmer reduces successful +rollout time. It does not keep the old process through initialization or startup, +does not guarantee that every layer remains present, and does not solve a bad +config or failed process. Keep it as an optional performance optimization. + +### N. In-Agent supervisor, socket activation, or file-descriptor transfer + +A long-lived supervisor can own listeners, download or select versioned Agent +binaries, launch a new child, and pass file descriptors with socket activation. +This can provide the most exact handoff and preserve socket objects. + +It moves image and process lifecycle outside ordinary Kubernetes container +semantics, complicating supply-chain policy, rollback, observability, and +resource isolation. A stable holder that runs as an explicit container is easier +to reason about. Keep this as a long-term Agent architecture option. + +## Decision matrix + +| Option | Old survives failed pull | New process starts first | Stable public endpoints | Request model | Change coverage | Complexity | +|---|---:|---:|---:|---|---|---| +| Current delete-first | No | No | No | One Pod | All template changes | Low | +| Image pre-pull only | Only before rollout | No | No | One Pod | Images | Low | +| Native surge + prepared Agent + handoff gate | Normal path | Yes | Same address/path, rebound at activation | Honest; temporarily two Pods | All Pod replacements | High | +| Stable endpoint holder + prepared Agents | Yes | Yes | Yes while holder lives | Honest; two Agents plus holder | All Agent replacements | High | +| Holder also owns Agent requests | Superficially | Yes | Yes | Incorrect cross-Pod accounting | All Agent replacements | Reject | +| Per-node placeholder + native surge | Yes | Yes | Same as prepared surge | Honest; permanently reserves overlap capacity | All Pod replacements | Medium/high cost | +| Small request then in-place resize | Conditional on resize | Yes | Same as prepared surge | Temporarily under-requested | CPU/memory and version dependent | High/experimental | +| OpenKruise Standard + prepared Agent | Yes | Yes | Same as native surge | Honest; temporarily two Pods | All Pod replacements | High dependency cost | +| OpenKruise in-place | No pull guarantee | No for changed container | Pod namespace persists | Honest; one Pod | Supported fields only; otherwise recreate | Medium/high | +| CSI alone | No | No | No | Unchanged | UDS mount lifecycle only | No useful solution alone | +| `SO_REUSEPORT` | Conditional | Yes | TCP/UDP only | Honest; temporarily two Pods | Does not provide active/passive handoff | Reject alone | +| Custom rollout controller + prepared Agent | Yes | Yes | Same as selected endpoint design | Honest; temporarily two Pods | All controlled changes | Very high | + +## Recommendation + +Continue with native DaemonSet surge, Agent prepared mode, and an +Active-acknowledged Operator handoff gate as the leading design because it +directly addresses the dominant delay without adding a permanent data-plane hop. +The small scheduler-compatibility PoC is implemented; the lifecycle and +coordinator are substantial, unimplemented work. + +Retain the current resource-fit classification logic, but place deletion behind +a separate explicit opt-in. Treat it as a best-effort, non-zero-gap escape hatch +for constrained nodes until capacity can be reserved atomically. + +Prototype two alternatives in parallel at small scope: + +1. OpenKruise `InPlaceIfPossible` for image-only and selected-container updates, + measuring the remaining restart gap and failed-pull behavior. +2. A minimal endpoint holder for DogStatsD UDP, APM TCP/HTTP, and stream/datagram + UDS, measuring origin metadata, connection drain, buffering, and inode + continuity. + +Do not move Agent requests to a different holder Pod. Evaluate the placeholder +DaemonSet only as an opt-in capacity policy for clusters willing to reserve +surge headroom permanently. + +## Security and operability + +Removing PodSpec port declarations does not reduce the privileges of a +host-network container. It only removes scheduler-visible reservations and API +metadata. Existing host-network, hostPath, kernel, and packet-capture risks +remain and should be documented independently. + +Prepared mode should reduce privileges before activation where practical, but a +single container cannot generally gain new Linux capabilities after startup. +The design must therefore treat the prepared process as privileged even while it +is sleeping and minimize its side effects. + +Ownership files require a root-owned host directory and stable permissions. A +lock key must identify the actual node-wide resource, such as protocol, address, +port, canonical UDS path, or kernel facility. Two installations cannot use +different lock keys to claim the same endpoint. Installation identity remains +useful for authorization and diagnostics, but not for weakening mutual +exclusion. Processes must not follow untrusted symlinks or replace lock files. +Endpoint holders require equivalent or greater hardening because they receive +all node-local telemetry and may preserve sender credentials. + +Mixed versions, rollback, node reboot, force deletion, kubelet restart, and +container-runtime cleanup need explicit tests. The safest response to an unknown +owner is to remain Prepared and report a blocking condition. + +## Observability + +The Operator and Agent should expose, per node and component: + +- rollout revision and old/replacement Pod UIDs; +- Starting, Prepared, Active, Draining, and fallback states; +- time spent pulling, initializing, prepared, waiting for ownership, activating, + and draining; +- ownership acquisition and release events; +- port or UDS bind failures and observed socket inode generation; +- resource-fallback candidates, reservations, deletions, and rejected reasons; +- active generation selected by an endpoint holder, if used; +- packets, requests, bytes, connections, queue depth, drops, and metadata loss + through a holder; and +- a per-node owner gauge that always alerts on multiple active owners and records + the duration of every zero-owner interval against the selected availability + policy. + +Pod Ready alone is not sufficient rollout telemetry. + +## Validation plan + +### Fast lab + +Use a two-worker Linux Kind cluster or KindVM and short teardown delays while +iterating. Docker Desktop is no longer available on the current workstation, so +local validation requires another container runtime or a remote Linux lab. + +Establish these baselines: + +1. Measure the current delete-first gap. +2. Show native surge works without host-network port reservations. +3. Show ordinary production `hostNetwork` plus declared ports blocks the second + Pod. +4. Show the prepared rendered template schedules two Pods while retaining + `hostNetwork: true`. +5. Reproduce a false-positive replacement probe against the old Agent's numeric + host-network health port, then show exec probes remove the alias. +6. Audit and exercise every production init container while the old Pod remains + Active. +7. Show a replacement that binds or unlinks production endpoints before + activation fails the safety tests. + +### Hypothesis-driven prototypes + +Run only the tests needed for the current question: + +- prepared Agent processes report Prepared without binding production + endpoints, while the unapproved Pod remains NotReady; +- the handoff coordinator approves no more than its budget and does not release + a token until all replacement components acknowledge Active; +- a long Terminating Pod cannot let the rollout accumulate unbounded handoffs; +- exec probes cannot accidentally interrogate the old host-network process; +- per-component locks allow trace/core activation while old system-probe still + drains; +- the Operator classifies only CPU or memory shortage, and a competing Pod test + demonstrates the non-atomic fallback race; +- the minimal endpoint holder preserves or intentionally translates source + metadata; and +- OpenKruise Standard and in-place strategies exhibit the documented failure + behavior. + +### Final validation for the leading design + +Use production Agent configuration and shared host resources. Generate numbered +signals so omissions and duplicates are visible: + +- DogStatsD metrics over UDP and UDS; +- APM traces over TCP/HTTP and supported UDS transports; +- OTLP gRPC and HTTP traffic; +- numbered log lines with restart and rotation cases; +- checks and process/network/security telemetry relevant to enabled components; + and +- long-lived and reconnecting socket clients. + +Exercise: + +- slow and failed image pulls; +- bad image references; +- init, configuration, startup, readiness, and activation failures; +- insufficient CPU, memory, pod count, and disk space; +- fallback enabled and disabled; +- image-only, configuration, resource, and mixed updates; +- Operator and API-server interruption; +- kubelet/container-runtime restart and Pod force deletion; +- rollback and mixed prepared-capable versions; and +- one real two-minute system-probe teardown while sibling containers exit. + +Pass phase one only if multiple Active owners are never observed, handoff and +fallback concurrency remain within their budgets, and every zero-owner interval +is attributable to the measured activation boundary or the explicitly enabled +fallback—not image pull, init, or prepared-process startup. Report numbered +metric, log, and trace omissions or duplicates rather than hiding them behind a +binary pass result. Strict zero-gap acceptance requires the endpoint-holder or +socket-handoff variant to demonstrate no zero-owner interval. Repeat the result +on Linux KindVM and an experimental cluster before defining a public Operator +API. + +## Open questions + +- Which initialization steps can each Agent binary safely complete before + activation? +- What are the correct ownership groups for core Agent, trace-agent, logs, + process-agent, system-probe, and security-agent? +- Can every network probe be replaced with a reliable exec probe without + changing existing health semantics? +- How should an active process prove a UDS pathname is safe to unlink after an + abnormal old-process exit? +- Which supported clients reconnect from old stream and datagram UDS socket + objects, and on what retry schedule? +- Do any internal or external Services rely on named Agent target ports? +- Is a one-time conventional bootstrap rollout acceptable, or must the first + prepared rollout interoperate with an old Agent that does not hold locks? +- Which configuration and resource changes can OpenKruise update in place for + the production multi-container Agent Pod? +- Is UDS inode continuity valuable enough to justify a permanent endpoint + holder? +- What authenticated status channel should carry Prepared, handoff-approved, + and per-component Active acknowledgements? + +## References + +- [Kubernetes `RollingUpdateDaemonSet` API](https://github.com/kubernetes/api/blob/v0.35.3/apps/v1/types.go#L609-L646) +- [Kubernetes host-network port defaulting](https://github.com/kubernetes/kubernetes/blob/v1.35.3/pkg/apis/core/v1/defaults.go#L396-L405) +- [Kubernetes scheduler host-port filter](https://github.com/kubernetes/kubernetes/blob/v1.35.3/pkg/scheduler/framework/plugins/nodeports/node_ports.go) +- [Kubernetes DaemonSet rolling-update implementation](https://github.com/kubernetes/kubernetes/blob/v1.35.3/pkg/controller/daemon/update.go) +- [Kubernetes DaemonSet per-node Pod management](https://github.com/kubernetes/kubernetes/blob/v1.35.3/pkg/controller/daemon/daemon_controller.go) +- [Kubernetes DaemonSet surge KEP](https://github.com/kubernetes/enhancements/tree/master/keps/sig-apps/1591-daemonset-surge) +- [Kubernetes Pod resource management](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) +- [Kubernetes Pod priority and preemption](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/) +- [Kubernetes in-place Pod resize](https://kubernetes.io/docs/tasks/configure-pod-container/resize-container-resources/) +- [Kubernetes Dynamic Resource Allocation](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/) +- [Kubernetes Service traffic policy](https://kubernetes.io/docs/concepts/services-networking/service-traffic-policy/) +- [Kubernetes CSI volumes](https://kubernetes.io/docs/concepts/storage/volumes/#csi) +- [OpenKruise v1.9 Advanced DaemonSet](https://github.com/openkruise/openkruise.io/blob/55e5c2228ac27026ced2ff1ec5384966cd59e71e/versioned_docs/version-v1.9/user-manuals/advanceddaemonset.md) +- [OpenKruise in-place update semantics](https://github.com/openkruise/openkruise.io/blob/55e5c2228ac27026ced2ff1ec5384966cd59e71e/docs/core-concepts/inplace-update.md) +- [OpenKruise ImagePullJob](https://openkruise.io/docs/user-manuals/imagepulljob) +- [OpenKruise Advanced DaemonSet rollout implementation](https://github.com/openkruise/kruise/blob/07169cfac7b9cf7800dda1b8652f850cc3184132/pkg/controller/daemonset/daemonset_update.go) +- [OpenKruise image pre-download implementation](https://github.com/openkruise/kruise/blob/07169cfac7b9cf7800dda1b8652f850cc3184132/pkg/controller/daemonset/daemonset_predownload_image.go) +- [`unix(7)` Unix-domain socket semantics](https://man7.org/linux/man-pages/man7/unix.7.html) +- [Prepared host-network surge PoC](agent_host_network_surge_poc.md) diff --git a/experiments/openkruise-prepull/README.md b/experiments/openkruise-prepull/README.md new file mode 100644 index 0000000000..d03222306a --- /dev/null +++ b/experiments/openkruise-prepull/README.md @@ -0,0 +1,378 @@ +# OpenKruise in-place update and image pre-pull experiment + +Run on 2026-07-22 against `kind-zero-gap-agent-rollout` with Kubernetes +v1.36.1 and OpenKruise v1.9.1. The workload uses one HTTP server and a +separate observer issuing numbered requests approximately every 100 ms. An +outage is reported as the interval between the last successful request before +an update and the first successful request after it. A failed request takes up +to one second because of the observer's timeout. + +This is a behavior experiment, not a performance benchmark. The images are +small and each timing case was run once. + +## Result + +`InPlaceIfPossible` preserved the Pod for image-only changes, but it did not +overlap old and new containers. The old container was stopped before a +replacement could be pulled and started. + +| Case | Pod identity | ImagePullJob | Observed result | +|---|---|---|---| +| Nonexistent image, no gate | UID stayed `48b32d66-337b-4945-832b-834e207757d9` | None | Old container exited and the Pod remained in `ImagePullBackOff` until manual rollback | +| Uncached `python:3.13-alpine`, no gate | Same UID; container ID changed to `8ecb557e...` | None | Kubelet pull took 2.559 s; success-to-success traffic gap was 7.845 s | +| Pre-pulled `python:3.14-alpine`, explicit gate | Same UID; container ID changed to `09725b13...` | `desired=1`, `succeeded=1`, `failed=0`, `active=0` before mutation | Image was already present; success-to-success traffic gap was 4.511 s | +| Nonexistent image, explicit gate | Pod UID, container ID `09725b13...`, image, and restart count stayed unchanged | `desired=1`, `succeeded=0`, `failed=1`, `active=0` | DaemonSet was not mutated and traffic remained successful | +| Pre-pull succeeds, process fails | Same UID; container ID changed to `32266554...` | Alpine pre-pull succeeded | Existing Python command exited 127; Pod entered `CrashLoopBackOff` and traffic stayed down until rollback | +| Unsupported environment change | UID changed from `48b32d66-...` to `02b00063-...` | Not applicable | `InPlaceIfPossible` fell back to Pod recreation; traffic gap was 5.635 s | + +The single, non-comparable runs observed gaps of 7.845 s and 4.511 s, a +3.333-second difference. The cached activation reported no kubelet pull, but +the runs used different tags and do not establish the delta as a causal effect +of pre-pulling. The cached run still had a container restart and readiness gap. + +Exact primary-workload identities, in observation order: + +```text +baseline UID: 48b32d66-337b-4945-832b-834e207757d9 +baseline container: c23b14b140f52e68af62f8f35c4dd6842b988ab28928677af7d12ded7957c7a1 +rollback container: f00d1aa7ef5ae397d8b97b6abff8f6e9d68d00b6fa47b16bc48fe474738b5310 +ungated container: 8ecb557e38acd3ad2b12c49c87e4e42229c70bbe17a131a9d14fb3b0b05e8e6c +pre-pulled container: 09725b13397863d1fbfc1725ddd95e45629082ecc044c8d98aa5abcb0698084f +failed-start container: 3226655429bfb858dcbd5e1ce355fd54b9a91c6bf8bd1a06b7b876626e07999b +pre-recreation container: 5153a3b55ea7771f9a00844b66b36b866ab392799d2fdc45c7bc368e5307676c +replacement UID: 02b00063-124a-4120-a241-9e46997cdc35 +replacement container: 5c3cb507bcc83554a76cd210b25719767c67f2e6b4022463253bf590989194cd +``` + +## Evidence + +Healthy baseline: + +```text +inplace-demo-l4mj7 48b32d66-337b-4945-832b-834e207757d9 true 0 +containerd://c23b14b1... python:3.12-alpine +``` + +For the ungated missing image, the update was submitted at `15:17:21Z`. +Kubelet reported that the old container finished at `15:17:23Z`, followed by +`ImagePullBackOff`. Observer transitions were: + +```text +2026-07-22T15:17:21.134632429Z 977 OK +2026-07-22T15:17:22.236060275Z 978 FAIL +2026-07-22T15:17:50.992993345Z 1005 OK # only after rollback +``` + +The uncached valid update was submitted at `15:18:32Z`. The kubelet event was: + +```text +Successfully pulled image "python:3.13-alpine" in 2.559s +``` + +Its request boundary was: + +```text +2026-07-22T15:18:33.409066694Z 1411 OK +2026-07-22T15:18:34.532342554Z 1412 FAIL +2026-07-22T15:18:41.253995616Z 1419 OK +``` + +The standalone successful gate ran before the DaemonSet mutation: + +```text +startTime: 2026-07-22T15:19:22Z +completionTime: 2026-07-22T15:19:27Z +desired: 1 +active: 0 +succeeded: 1 +failed: 0 +``` + +After activation, kubelet reported the image was already present. Its request +boundary was: + +```text +2026-07-22T15:19:44.684584633Z 2036 OK +2026-07-22T15:19:45.786420014Z 2037 FAIL +2026-07-22T15:19:49.196018154Z 2041 OK +``` + +The standalone failed gate completed in five seconds: + +```text +startTime: 2026-07-22T15:20:27Z +completionTime: 2026-07-22T15:20:32Z +desired: 1 +active: 0 +succeeded: 0 +failed: 1 +failedNodes: [zero-gap-agent-rollout-worker] +``` + +Because the experiment did not patch the DaemonSet after that result, the Pod +remained Ready with the same UID, container ID, restart count, and image. The +next twenty observer requests were all successful. + +The activation-failure job successfully cached `alpine:3.22` at `15:21:00Z`. +After activation, the unchanged `python -m http.server` command exited 127 and +the first failed request followed the last success: + +```text +2026-07-22T15:21:19.211459847Z 2922 OK +2026-07-22T15:21:20.313133586Z 2923 FAIL +``` + +Adding an environment variable is not an eligible in-place image change. The +old Pod was deleted and the replacement became ready with a new UID. The +request boundary was: + +```text +2026-07-22T15:22:10.048879229Z 3087 OK +2026-07-22T15:22:11.150162924Z 3088 FAIL +2026-07-22T15:22:15.683683204Z 3093 OK +``` + +## Built-in AdvancedDaemonSet pre-download + +OpenKruise v1.9.1 ships `PreDownloadImageForDaemonSetUpdate` as an alpha +feature gate that defaults to false. Merely enabling `ImagePullJobGate` runs +standalone jobs but does not enable automatic AdvancedDaemonSet pre-download. + +The automatic code also skips pre-download when all Pods can update in one +batch. A separate two-worker DaemonSet with `maxUnavailable: 1` was therefore +used. After temporarily starting the managers with: + +```text +--feature-gates=ImagePullJobGate=true,PreDownloadImageForDaemonSetUpdate=true +``` + +an update to a nonexistent image produced the owned job +`automatic-predownload-demo-544cd9c95d-server` at `15:26:32Z`. One second +later the job was active and both Pods were still Ready. The controller did not +wait for it: the selected Pod's old container exited at `15:26:34Z`. At +`15:26:41Z` the job had `active=1`, `failed=1`, while that Pod was already in +`ImagePullBackOff`. The other Pod stayed healthy only because +`maxUnavailable: 1` stopped the rollout globally; the per-node availability +invariant was violated on the updated node. + +The feature flag was restored to its original value after the test: + +```text +--feature-gates=ImagePullJobGate=true +``` + +## Reproduction + +Prerequisites are the `kind-zero-gap-agent-rollout` context, two schedulable +workers named `zero-gap-agent-rollout-worker` and +`zero-gap-agent-rollout-worker2`, OpenKruise v1.9.1 with its manager and node +daemon healthy, and registry access for the fixture images. The checked-in +selectors intentionally depend on those worker names. + +```sh +# Reset only the isolated experiment namespace. +kubectl --context kind-zero-gap-agent-rollout delete namespace \ + openkruise-prepull-lab --ignore-not-found --wait=true +kubectl --context kind-zero-gap-agent-rollout apply \ + -f experiments/openkruise-prepull/base.yaml +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab wait --for=condition=Ready pod \ + -l app=inplace-demo --timeout=180s + +# Missing image without a gate; inspect ImagePullBackOff, then roll back. +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab patch daemonset.apps.kruise.io inplace-demo \ + --type=json \ + -p='[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"python:zero-gap-tag-does-not-exist"}]' +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab wait \ + --for=jsonpath='{.status.containerStatuses[0].state.waiting.reason}'=ImagePullBackOff \ + pod -l app=inplace-demo --timeout=180s +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab get pod -l app=inplace-demo -o wide +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab patch daemonset.apps.kruise.io inplace-demo \ + --type=json \ + -p='[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"python:3.12-alpine"}]' +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab wait \ + --for=jsonpath='{.status.containerStatuses[0].image}'=docker.io/library/python:3.12-alpine \ + pod -l app=inplace-demo --timeout=180s +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab wait --for=condition=Ready pod \ + -l app=inplace-demo --timeout=180s + +# Ungated image-only update +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab patch daemonset.apps.kruise.io inplace-demo \ + --type=json \ + -p='[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"python:3.13-alpine"}]' +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab wait \ + --for=jsonpath='{.status.containerStatuses[0].image}'=docker.io/library/python:3.13-alpine \ + pod -l app=inplace-demo --timeout=180s +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab wait --for=condition=Ready pod \ + -l app=inplace-demo --timeout=180s + +# Explicit pre-pull gate +kubectl --context kind-zero-gap-agent-rollout apply \ + -f experiments/openkruise-prepull/imagepulljob-good.yaml +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab get imagepulljob python-3-14-alpine -o yaml +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab wait \ + --for=jsonpath='{.status.completionTime}' imagepulljob/python-3-14-alpine \ + --timeout=360s +test "$(kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab get imagepulljob python-3-14-alpine \ + -o jsonpath='{.status.desired},{.status.succeeded},{.status.failed},{.status.active}')" \ + = "1,1,0,0" + +# Mutate the DaemonSet only after desired > 0, succeeded == desired, +# failed == 0, active == 0, and completionTime is set. +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab patch daemonset.apps.kruise.io inplace-demo \ + --type=json \ + -p='[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"python:3.14-alpine"}]' +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab wait \ + --for=jsonpath='{.status.containerStatuses[0].image}'=docker.io/library/python:3.14-alpine \ + pod -l app=inplace-demo --timeout=180s +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab wait --for=condition=Ready pod \ + -l app=inplace-demo --timeout=180s + +# Failed gate: inspect the result and deliberately do not patch the DaemonSet. +kubectl --context kind-zero-gap-agent-rollout apply \ + -f experiments/openkruise-prepull/imagepulljob-bad.yaml +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab get imagepulljob python-missing -o yaml +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab wait \ + --for=jsonpath='{.status.completionTime}' imagepulljob/python-missing \ + --timeout=120s +test "$(kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab get imagepulljob python-missing \ + -o jsonpath='{.status.desired},{.status.succeeded},{.status.failed},{.status.active}')" \ + = "1,0,1,0" + +# Successfully cached image with an invalid retained command. +kubectl --context kind-zero-gap-agent-rollout apply \ + -f experiments/openkruise-prepull/imagepulljob-activation-failure.yaml +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab wait \ + --for=jsonpath='{.status.completionTime}' imagepulljob/alpine-3-22 \ + --timeout=360s +test "$(kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab get imagepulljob alpine-3-22 \ + -o jsonpath='{.status.desired},{.status.succeeded},{.status.failed},{.status.active}')" \ + = "1,1,0,0" +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab patch daemonset.apps.kruise.io inplace-demo \ + --type=json \ + -p='[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"alpine:3.22"}]' +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab wait \ + --for=jsonpath='{.status.containerStatuses[0].state.waiting.reason}'=CrashLoopBackOff \ + pod -l app=inplace-demo --timeout=180s +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab patch daemonset.apps.kruise.io inplace-demo \ + --type=json \ + -p='[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"python:3.14-alpine"}]' +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab wait \ + --for=jsonpath='{.status.containerStatuses[0].image}'=docker.io/library/python:3.14-alpine \ + pod -l app=inplace-demo --timeout=180s +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab wait --for=condition=Ready pod \ + -l app=inplace-demo --timeout=180s + +# Unsupported Pod-template change. Record and require a new Pod UID. +inplace_uid=$(kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab get pod -l app=inplace-demo \ + -o jsonpath='{.items[0].metadata.uid}') +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab patch daemonset.apps.kruise.io inplace-demo \ + --type=json \ + -p='[{"op":"add","path":"/spec/template/spec/containers/0/env","value":[{"name":"UNSUPPORTED_TEMPLATE_CHANGE","value":"true"}]}]' +while test "$(kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab get pod -l app=inplace-demo \ + -o jsonpath='{.items[0].metadata.uid}' 2>/dev/null)" = "$inplace_uid"; do sleep 1; done +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab wait --for=condition=Ready pod \ + -l app=inplace-demo --timeout=180s + +# Two-worker automatic pre-download fixture. +kubectl --context kind-zero-gap-agent-rollout apply \ + -f experiments/openkruise-prepull/automatic-predownload.yaml +# Do not continue until both worker Pods are Ready. +while test "$(kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab get daemonset.apps.kruise.io \ + automatic-predownload-demo -o jsonpath='{.status.numberReady}')" != "2"; do sleep 1; done +# Verify the current argument before using the observed index from this install. +kubectl --context kind-zero-gap-agent-rollout -n kruise-system get deployment \ + kruise-controller-manager \ + -o jsonpath='{.spec.template.spec.containers[0].args[6]}{"\n"}' +# Abort unless the preceding command prints exactly: +# --feature-gates=ImagePullJobGate=true +kubectl --context kind-zero-gap-agent-rollout -n kruise-system patch deployment \ + kruise-controller-manager --type=json \ + -p='[{"op":"replace","path":"/spec/template/spec/containers/0/args/6","value":"--feature-gates=ImagePullJobGate=true,PreDownloadImageForDaemonSetUpdate=true"}]' +kubectl --context kind-zero-gap-agent-rollout -n kruise-system rollout status \ + deployment/kruise-controller-manager --timeout=180s +kubectl --context kind-zero-gap-agent-rollout -n kruise-system get deployment \ + kruise-controller-manager \ + -o jsonpath='{.spec.template.spec.containers[0].args[6]}{"\n"}' +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab patch daemonset.apps.kruise.io \ + automatic-predownload-demo --type=json \ + -p='[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"python:zero-gap-auto-gate-does-not-exist"}]' +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab get imagepulljobs.apps.kruise.io +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab get pods -l app=automatic-predownload-demo -o wide + +# Observer transition extraction used for each timing boundary. +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab logs observer --timestamps \ + --since-time=2026-07-22T15:19:43Z \ + | awk '/ (OK|FAIL)$/ {state=$NF; if (state != previous) {print; previous=state}}' + +# Restore the two-worker fixture and original manager feature flags. +kubectl --context kind-zero-gap-agent-rollout apply \ + -f experiments/openkruise-prepull/automatic-predownload.yaml +while test "$(kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab get daemonset.apps.kruise.io \ + automatic-predownload-demo -o jsonpath='{.status.numberReady}')" != "2"; do sleep 1; done +kubectl --context kind-zero-gap-agent-rollout -n kruise-system patch deployment \ + kruise-controller-manager --type=json \ + -p='[{"op":"replace","path":"/spec/template/spec/containers/0/args/6","value":"--feature-gates=ImagePullJobGate=true"}]' +kubectl --context kind-zero-gap-agent-rollout -n kruise-system rollout status \ + deployment/kruise-controller-manager --timeout=180s +kubectl --context kind-zero-gap-agent-rollout apply \ + -f experiments/openkruise-prepull/base.yaml +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab wait \ + --for=jsonpath='{.status.containerStatuses[0].image}'=docker.io/library/python:3.12-alpine \ + pod -l app=inplace-demo --timeout=180s +kubectl --context kind-zero-gap-agent-rollout \ + -n openkruise-prepull-lab get pods -o wide +kubectl --context kind-zero-gap-agent-rollout -n kruise-system get deployment \ + kruise-controller-manager \ + -o jsonpath='{.spec.template.spec.containers[0].args[6]}{"\n"}' +``` + +For a production gate, use an immutable digest with `IfNotPresent`. A mutable +tag or `Always` can still require registry resolution during activation. Image +garbage collection or a node restart between gate completion and activation +can also evict the cached digest, so this is not a permanent pull guarantee. + +## Conclusion + +Image pre-pulling is worthwhile preparation. The built-in automatic mechanism +is not an availability gate, while a standalone Operator-managed ImagePullJob +can gate DaemonSet mutation on a successful pull while that cached image +remains available. Neither form validates process startup, and in-place update +still has a restart gap. It is therefore a complementary optimization, not the +zero-gap primitive. diff --git a/experiments/openkruise-prepull/automatic-predownload.yaml b/experiments/openkruise-prepull/automatic-predownload.yaml new file mode 100644 index 0000000000..4f14568abd --- /dev/null +++ b/experiments/openkruise-prepull/automatic-predownload.yaml @@ -0,0 +1,43 @@ +apiVersion: apps.kruise.io/v1beta1 +kind: DaemonSet +metadata: + name: automatic-predownload-demo + namespace: openkruise-prepull-lab +spec: + selector: + matchLabels: + app: automatic-predownload-demo + updateStrategy: + type: RollingUpdate + rollingUpdate: + rollingUpdateType: InPlaceIfPossible + maxUnavailable: 1 + template: + metadata: + labels: + app: automatic-predownload-demo + spec: + terminationGracePeriodSeconds: 1 + containers: + - name: server + image: python:3.12-alpine + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - | + mkdir -p /www + printf 'ok\n' >/www/index.html + exec python -m http.server 8080 --directory /www + readinessProbe: + httpGet: + path: / + port: 8080 + periodSeconds: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 10m + memory: 8Mi + limits: + memory: 32Mi diff --git a/experiments/openkruise-prepull/base.yaml b/experiments/openkruise-prepull/base.yaml new file mode 100644 index 0000000000..fde53ee5db --- /dev/null +++ b/experiments/openkruise-prepull/base.yaml @@ -0,0 +1,100 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: openkruise-prepull-lab +--- +apiVersion: apps.kruise.io/v1beta1 +kind: DaemonSet +metadata: + name: inplace-demo + namespace: openkruise-prepull-lab +spec: + selector: + matchLabels: + app: inplace-demo + updateStrategy: + type: RollingUpdate + rollingUpdate: + rollingUpdateType: InPlaceIfPossible + maxUnavailable: 1 + template: + metadata: + labels: + app: inplace-demo + spec: + nodeSelector: + kubernetes.io/hostname: zero-gap-agent-rollout-worker + terminationGracePeriodSeconds: 1 + containers: + - name: server + image: python:3.12-alpine + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - | + mkdir -p /www + printf 'ok\n' >/www/index.html + exec python -m http.server 8080 --directory /www + ports: + - name: http + containerPort: 8080 + readinessProbe: + httpGet: + path: / + port: 8080 + periodSeconds: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 10m + memory: 8Mi + limits: + memory: 32Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: inplace-demo + namespace: openkruise-prepull-lab +spec: + selector: + app: inplace-demo + ports: + - name: http + port: 80 + targetPort: http +--- +apiVersion: v1 +kind: Pod +metadata: + name: observer + namespace: openkruise-prepull-lab +spec: + nodeSelector: + kubernetes.io/hostname: zero-gap-agent-rollout-worker2 + restartPolicy: Never + containers: + - name: observer + image: alpine:3.20 + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - | + sequence=0 + while true; do + sequence=$((sequence + 1)) + if wget -q -T 1 -O /dev/null http://inplace-demo; then + printf '%d OK\n' "${sequence}" + else + printf '%d FAIL\n' "${sequence}" + fi + sleep 0.1 + done + resources: + requests: + cpu: 5m + memory: 4Mi + limits: + memory: 16Mi diff --git a/experiments/openkruise-prepull/imagepulljob-activation-failure.yaml b/experiments/openkruise-prepull/imagepulljob-activation-failure.yaml new file mode 100644 index 0000000000..1f6cc89437 --- /dev/null +++ b/experiments/openkruise-prepull/imagepulljob-activation-failure.yaml @@ -0,0 +1,18 @@ +apiVersion: apps.kruise.io/v1alpha1 +kind: ImagePullJob +metadata: + name: alpine-3-22 + namespace: openkruise-prepull-lab +spec: + image: alpine:3.22 + imagePullPolicy: IfNotPresent + selector: + matchLabels: + kubernetes.io/hostname: zero-gap-agent-rollout-worker + parallelism: 1 + pullPolicy: + timeoutSeconds: 300 + backoffLimit: 1 + completionPolicy: + type: Always + activeDeadlineSeconds: 360 diff --git a/experiments/openkruise-prepull/imagepulljob-bad.yaml b/experiments/openkruise-prepull/imagepulljob-bad.yaml new file mode 100644 index 0000000000..793b6cd5ef --- /dev/null +++ b/experiments/openkruise-prepull/imagepulljob-bad.yaml @@ -0,0 +1,18 @@ +apiVersion: apps.kruise.io/v1alpha1 +kind: ImagePullJob +metadata: + name: python-missing + namespace: openkruise-prepull-lab +spec: + image: python:zero-gap-tag-does-not-exist + imagePullPolicy: Always + selector: + matchLabels: + kubernetes.io/hostname: zero-gap-agent-rollout-worker + parallelism: 1 + pullPolicy: + timeoutSeconds: 60 + backoffLimit: 0 + completionPolicy: + type: Always + activeDeadlineSeconds: 90 diff --git a/experiments/openkruise-prepull/imagepulljob-good.yaml b/experiments/openkruise-prepull/imagepulljob-good.yaml new file mode 100644 index 0000000000..bfdb8ca6c4 --- /dev/null +++ b/experiments/openkruise-prepull/imagepulljob-good.yaml @@ -0,0 +1,18 @@ +apiVersion: apps.kruise.io/v1alpha1 +kind: ImagePullJob +metadata: + name: python-3-14-alpine + namespace: openkruise-prepull-lab +spec: + image: python:3.14-alpine + imagePullPolicy: IfNotPresent + selector: + matchLabels: + kubernetes.io/hostname: zero-gap-agent-rollout-worker + parallelism: 1 + pullPolicy: + timeoutSeconds: 300 + backoffLimit: 1 + completionPolicy: + type: Always + activeDeadlineSeconds: 360 diff --git a/internal/controller/datadogagentinternal/controller_reconcile_agent.go b/internal/controller/datadogagentinternal/controller_reconcile_agent.go index 1c700adcc4..51412d3af8 100644 --- a/internal/controller/datadogagentinternal/controller_reconcile_agent.go +++ b/internal/controller/datadogagentinternal/controller_reconcile_agent.go @@ -12,6 +12,7 @@ import ( edsv1alpha1 "github.com/DataDog/extendeddaemonset/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" @@ -125,8 +126,8 @@ func (r *Reconciler) reconcileV2Agent(ctx context.Context, requiredComponents fe true, ) } - if err := r.deleteV2ExtendedDaemonSet(ctx, ddai, eds, newStatus); err != nil { - return reconcile.Result{}, err + if deleteErr := r.deleteV2ExtendedDaemonSet(ctx, ddai, eds, newStatus); deleteErr != nil { + return reconcile.Result{}, deleteErr } return reconcile.Result{}, nil } @@ -219,8 +220,8 @@ func (r *Reconciler) reconcileV2Agent(ctx context.Context, requiredComponents fe daemonsetLogger.Info("Removing Windows DaemonSet: FIPS (useFIPSAgent or fips.enabled) is enabled and FIPS is unsupported on Windows") // Delete any existing Windows DaemonSet so a non-FIPS agent does not keep running // under a FIPS-required configuration (compliance), rather than silently downgrading. - if err := r.deleteV2DaemonSet(ctx, ddai, daemonset, newStatus); err != nil { - return reconcile.Result{}, err + if deleteErr := r.deleteV2DaemonSet(ctx, ddai, daemonset, newStatus); deleteErr != nil { + return reconcile.Result{}, deleteErr } // Clear the Agent status unconditionally: deleteV2DaemonSet returns early (without // clearing status) if the DaemonSet is already gone, which would otherwise leave a @@ -283,25 +284,43 @@ func (r *Reconciler) reconcileV2Agent(ctx context.Context, requiredComponents fe true, ) } - if err := r.deleteV2DaemonSet(ctx, ddai, daemonset, newStatus); err != nil { - return reconcile.Result{}, err + if deleteErr := r.deleteV2DaemonSet(ctx, ddai, daemonset, newStatus); deleteErr != nil { + return reconcile.Result{}, deleteErr } deleteStatusWithAgent(newStatus) return reconcile.Result{}, nil } - fallbackBudget := resourceFallbackBudget(ddai, &r.options.ExtendedDaemonsetOptions) - fallbackEnabled := configureResourceFallback(daemonset, fallbackBudget) + rolloutBudget := resourceFallbackBudget(ddai, &r.options.ExtendedDaemonsetOptions) + preparedPhase, prepareErr := r.configurePreparedRollout(ctx, ddai, daemonset, rolloutBudget) + if prepareErr != nil { + objLogger.Error(prepareErr, "Prepared Agent rollout request is incompatible with the rendered Pod template") + if r.recorder != nil { + r.recorder.Eventf(ddai, corev1.EventTypeWarning, "AgentPreparedRolloutRejected", "Prepared Agent rollout is disabled for this template: %v", prepareErr) + } + return reconcile.Result{}, prepareErr + } result, err = r.createOrUpdateDaemonset(ctx, ddai, daemonset, newStatus, updateDSStatusV2WithAgent) - if err != nil || !fallbackEnabled { + if err != nil || preparedPhase == "" { return result, err } - fallbackResult, err := r.reconcileResourceFallback(ctx, ddai, daemonset, fallbackBudget) - if err != nil { - return reconcile.Result{}, err + if preparedPhase == preparedRolloutPhaseStandby { + handoffResult, handoffErr := r.reconcilePreparedHandoff(ctx, ddai, daemonset, rolloutBudget) + if handoffErr != nil { + return reconcile.Result{}, handoffErr + } + if handoffResult.RequeueAfter > 0 && (result.RequeueAfter == 0 || handoffResult.RequeueAfter < result.RequeueAfter) { + result.RequeueAfter = handoffResult.RequeueAfter + } } - if fallbackResult.RequeueAfter > 0 && (result.RequeueAfter == 0 || fallbackResult.RequeueAfter < result.RequeueAfter) { - result.RequeueAfter = fallbackResult.RequeueAfter + if resourceFallbackEnabled(ddai) { + fallbackResult, fallbackErr := r.reconcileResourceFallback(ctx, ddai, daemonset, rolloutBudget) + if fallbackErr != nil { + return reconcile.Result{}, fallbackErr + } + if fallbackResult.RequeueAfter > 0 && (result.RequeueAfter == 0 || fallbackResult.RequeueAfter < result.RequeueAfter) { + result.RequeueAfter = fallbackResult.RequeueAfter + } } return result, nil } diff --git a/internal/controller/datadogagentinternal/prepared_rollout.go b/internal/controller/datadogagentinternal/prepared_rollout.go new file mode 100644 index 0000000000..fc0773ebc1 --- /dev/null +++ b/internal/controller/datadogagentinternal/prepared_rollout.go @@ -0,0 +1,447 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// Copyright 2016-present Datadog, Inc. + +package datadogagentinternal + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + apicommon "github.com/DataDog/datadog-operator/api/datadoghq/common" + datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" +) + +const ( + preparedRolloutAnnotation = "experimental.agent.datadoghq.com/host-network-surge-prepared" + preparedRolloutPhaseAnnotation = "experimental.agent.datadoghq.com/prepared-rollout-phase" + resourceFallbackAnnotation = "experimental.agent.datadoghq.com/resource-fallback" + preparedRolloutPhaseArm = "arm" + preparedRolloutPhaseStandby = "standby" + + preparedRolloutLockVolume = "agent-rollout-locks" + preparedRolloutStateVolume = "agent-rollout-state" + preparedRolloutLockDir = "/var/run/datadog-agent-rollout" + preparedRolloutStateDir = "/var/run/datadog-agent-rollout-state" + + rolloutEnabledEnv = "DD_EXPERIMENTAL_NODE_AGENT_ROLLOUT_ENABLED" + rolloutLockPathEnv = "DD_EXPERIMENTAL_NODE_AGENT_ROLLOUT_LOCK_PATH" + rolloutStatePathEnv = "DD_EXPERIMENTAL_NODE_AGENT_ROLLOUT_STATE_PATH" +) + +var preparedRolloutContainerNames = []string{ + string(apicommon.CoreAgentContainerName), + string(apicommon.TraceAgentContainerName), +} + +func preparedRolloutEnabled(ddai *datadoghqv1alpha1.DatadogAgentInternal) bool { + return strings.EqualFold(ddai.Annotations[preparedRolloutAnnotation], "true") +} + +func resourceFallbackEnabled(ddai *datadoghqv1alpha1.DatadogAgentInternal) bool { + return preparedRolloutEnabled(ddai) && strings.EqualFold(ddai.Annotations[resourceFallbackAnnotation], "true") +} + +// configurePreparedRollout installs a restart-safe two-phase protocol. A +// conventional rollout first arms every old process with the node-local lock. +// Only an exact, fully available arm revision may transition to standby surge. +func (r *Reconciler) configurePreparedRollout(ctx context.Context, ddai *datadoghqv1alpha1.DatadogAgentInternal, desired *appsv1.DaemonSet, budget intstr.IntOrString) (string, error) { + if !preparedRolloutEnabled(ddai) { + return "", nil + } + if !positiveIntOrPercent(&budget) { + return "", fmt.Errorf("prepared Agent rollout requires a positive, valid maxUnavailable budget") + } + if desired.Spec.UpdateStrategy.Type != "" && desired.Spec.UpdateStrategy.Type != appsv1.RollingUpdateDaemonSetStrategyType { + return "", fmt.Errorf("prepared Agent rollout requires RollingUpdate strategy") + } + + armed := desired.DeepCopy() + if err := prepareAgentTemplate(armed, preparedRolloutPhaseArm); err != nil { + return "", err + } + configureArmStrategy(armed, budget) + + live := &appsv1.DaemonSet{} + err := r.apiReader.Get(ctx, client.ObjectKeyFromObject(desired), live) + if err != nil && !apierrors.IsNotFound(err) { + return "", fmt.Errorf("get live Agent DaemonSet for prepared rollout: %w", err) + } + + phase := preparedRolloutPhaseArm + if err == nil { + livePhase := live.Spec.Template.Annotations[preparedRolloutPhaseAnnotation] + switch livePhase { + case preparedRolloutPhaseStandby: + // Never oscillate back to arm during a mixed or failed surged rollout. + phase = preparedRolloutPhaseStandby + case preparedRolloutPhaseArm: + if apiequality.Semantic.DeepEqual(live.Spec.Template, armed.Spec.Template) && daemonSetArmComplete(live) { + phase = preparedRolloutPhaseStandby + } + } + } + + if phase == preparedRolloutPhaseArm { + *desired = *armed + return phase, nil + } + + standby := desired.DeepCopy() + if err := prepareAgentTemplate(standby, preparedRolloutPhaseStandby); err != nil { + return "", err + } + if !configureResourceFallback(standby, budget) { + return "", fmt.Errorf("prepared Agent rollout requires a positive, valid maxUnavailable budget") + } + *desired = *standby + return phase, nil +} + +func daemonSetArmComplete(ds *appsv1.DaemonSet) bool { + desired := ds.Status.DesiredNumberScheduled + return desired > 0 && + ds.Status.ObservedGeneration == ds.Generation && + ds.Status.UpdatedNumberScheduled == desired && + ds.Status.NumberReady == desired && + ds.Status.NumberAvailable == desired && + ds.Status.NumberUnavailable == 0 +} + +func configureArmStrategy(ds *appsv1.DaemonSet, budget intstr.IntOrString) { + ds.Spec.UpdateStrategy.Type = appsv1.RollingUpdateDaemonSetStrategyType + if ds.Spec.UpdateStrategy.RollingUpdate == nil { + ds.Spec.UpdateStrategy.RollingUpdate = &appsv1.RollingUpdateDaemonSet{} + } + zero := intstr.FromInt(0) + ds.Spec.UpdateStrategy.RollingUpdate.MaxSurge = &zero + if positiveIntOrPercent(&budget) { + value := budget + ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable = &value + } +} + +func prepareAgentTemplate(ds *appsv1.DaemonSet, phase string) error { + spec := &ds.Spec.Template.Spec + if !spec.HostNetwork { + return fmt.Errorf("prepared Agent rollout requires hostNetwork=true") + } + if spec.OS != nil && spec.OS.Name != corev1.Linux { + return fmt.Errorf("prepared Agent rollout is Linux-only") + } + if spec.NodeSelector[corev1.LabelOSStable] == "windows" || spec.NodeSelector["beta.kubernetes.io/os"] == "windows" { + return fmt.Errorf("prepared Agent rollout is Linux-only") + } + if err := validatePreparedContainers(spec); err != nil { + return err + } + if !prepareProfileAntiAffinityForSurge(&ds.Spec.Template) { + return fmt.Errorf("prepared Agent rollout does not support custom Pod anti-affinity") + } + if err := addPreparedRolloutVolumes(spec); err != nil { + return err + } + + for i := range spec.Containers { + container := &spec.Containers[i] + if container.Name == string(apicommon.TraceAgentContainerName) { + traceIndex := -1 + for commandIndex, command := range container.Command { + if command == "trace-agent" { + traceIndex = commandIndex + break + } + } + if traceIndex < 0 { + return fmt.Errorf("prepared Agent rollout cannot bypass an unknown trace-agent loader command") + } + container.Command = append([]string(nil), container.Command[traceIndex:]...) + } + configurePreparedContainer(container) + if phase == preparedRolloutPhaseStandby { + container.Ports = nil + } + } + if phase == preparedRolloutPhaseStandby { + for i := range spec.InitContainers { + spec.InitContainers[i].Ports = nil + } + } + if ds.Spec.Template.Annotations == nil { + ds.Spec.Template.Annotations = map[string]string{} + } + ds.Spec.Template.Annotations[preparedRolloutPhaseAnnotation] = phase + return nil +} + +func validatePreparedContainers(spec *corev1.PodSpec) error { + if len(spec.Containers) != len(preparedRolloutContainerNames) { + return fmt.Errorf("prepared Agent rollout initially supports exactly agent and trace-agent containers") + } + seen := map[string]bool{} + for i := range spec.Containers { + container := &spec.Containers[i] + if container.Name != string(apicommon.CoreAgentContainerName) && container.Name != string(apicommon.TraceAgentContainerName) { + return fmt.Errorf("prepared Agent rollout does not support container %q", container.Name) + } + if seen[container.Name] { + return fmt.Errorf("prepared Agent rollout found duplicate container %q", container.Name) + } + seen[container.Name] = true + if container.Lifecycle != nil { + return fmt.Errorf("prepared Agent rollout does not support lifecycle hooks on container %q", container.Name) + } + if len(container.Args) != 0 { + return fmt.Errorf("prepared Agent rollout does not support command arguments on container %q", container.Name) + } + if container.Name == string(apicommon.CoreAgentContainerName) && (len(container.Command) != 2 || container.Command[0] != "agent" || container.Command[1] != "run") { + return fmt.Errorf("prepared Agent rollout requires the standard agent run command") + } + for _, mount := range container.VolumeMounts { + if mount.Name == preparedRolloutLockVolume || mount.Name == preparedRolloutStateVolume || mount.MountPath == preparedRolloutLockDir || mount.MountPath == preparedRolloutStateDir { + return fmt.Errorf("prepared Agent rollout volume mount on container %q conflicts with a reserved name or path", container.Name) + } + } + } + if !seen[string(apicommon.CoreAgentContainerName)] || !seen[string(apicommon.TraceAgentContainerName)] { + return fmt.Errorf("prepared Agent rollout requires agent and trace-agent containers") + } + if len(spec.InitContainers) != 2 { + return fmt.Errorf("prepared Agent rollout initially supports only init-volume and init-config init containers") + } + seenInit := map[string]bool{} + for i := range spec.InitContainers { + container := &spec.InitContainers[i] + if container.Name != string(apicommon.InitVolumeContainerName) && container.Name != string(apicommon.InitConfigContainerName) { + return fmt.Errorf("prepared Agent rollout does not support init container %q", container.Name) + } + if container.Lifecycle != nil || len(container.Ports) != 0 { + return fmt.Errorf("prepared Agent rollout does not support ports or lifecycle hooks on init container %q", container.Name) + } + seenInit[container.Name] = true + } + if !seenInit[string(apicommon.InitVolumeContainerName)] || !seenInit[string(apicommon.InitConfigContainerName)] { + return fmt.Errorf("prepared Agent rollout requires init-volume and init-config") + } + return nil +} + +func addPreparedRolloutVolumes(spec *corev1.PodSpec) error { + for i := range spec.Volumes { + if spec.Volumes[i].Name == preparedRolloutLockVolume || spec.Volumes[i].Name == preparedRolloutStateVolume { + return fmt.Errorf("prepared Agent rollout volume name %q is reserved", spec.Volumes[i].Name) + } + } + directoryOrCreate := corev1.HostPathDirectoryOrCreate + spec.Volumes = append(spec.Volumes, + corev1.Volume{ + Name: preparedRolloutLockVolume, + VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ + Path: preparedRolloutLockDir, + Type: &directoryOrCreate, + }}, + }, + corev1.Volume{ + Name: preparedRolloutStateVolume, + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + }, + ) + return nil +} + +func configurePreparedContainer(container *corev1.Container) { + lockPath := preparedRolloutLockDir + "/" + container.Name + ".lock" + statePath := preparedRolloutStateDir + "/" + container.Name + ".state" + setContainerEnv(container, rolloutEnabledEnv, "true") + setContainerEnv(container, rolloutLockPathEnv, lockPath) + setContainerEnv(container, rolloutStatePathEnv, statePath) + container.VolumeMounts = append(container.VolumeMounts, + corev1.VolumeMount{Name: preparedRolloutLockVolume, MountPath: preparedRolloutLockDir}, + corev1.VolumeMount{Name: preparedRolloutStateVolume, MountPath: preparedRolloutStateDir}, + ) + container.StartupProbe = rolloutStateProbe(statePath, "prepared|activating|active", 1, 300) + container.LivenessProbe = rolloutStateProbe(statePath, "prepared|activating|active", 10, 3) + container.ReadinessProbe = rolloutStateProbe(statePath, "active", 1, 3) +} + +func setContainerEnv(container *corev1.Container, name, value string) { + for i := range container.Env { + if container.Env[i].Name == name { + container.Env[i] = corev1.EnvVar{Name: name, Value: value} + return + } + } + container.Env = append(container.Env, corev1.EnvVar{Name: name, Value: value}) +} + +func rolloutStateProbe(path, accepted string, period, failures int32) *corev1.Probe { + command := fmt.Sprintf(`case "$(cat %s 2>/dev/null)" in %s) exit 0;; *) exit 1;; esac`, path, accepted) + return &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"sh", "-c", command}}}, + PeriodSeconds: period, + TimeoutSeconds: 1, + FailureThreshold: failures, + } +} + +type preparedHandoffCandidate struct { + replacement *corev1.Pod + old *corev1.Pod + nodeName string + reserved bool +} + +func (r *Reconciler) reconcilePreparedHandoff(ctx context.Context, ddai *datadoghqv1alpha1.DatadogAgentInternal, expectedDS *appsv1.DaemonSet, budgetValue intstr.IntOrString) (reconcile.Result, error) { + reader := r.apiReader + liveDS := &appsv1.DaemonSet{} + if err := reader.Get(ctx, client.ObjectKeyFromObject(expectedDS), liveDS); err != nil { + return reconcile.Result{}, client.IgnoreNotFound(err) + } + if !daemonSetControlledByDDAI(liveDS, ddai) || liveDS.Spec.Template.Annotations[preparedRolloutPhaseAnnotation] != preparedRolloutPhaseStandby || !resourceFallbackDaemonSetEligible(liveDS) { + return reconcile.Result{}, nil + } + currentRevision, err := currentDaemonSetRevision(ctx, reader, liveDS) + if err != nil || currentRevision == "" { + return reconcile.Result{}, err + } + pods, err := daemonSetPods(ctx, reader, liveDS) + if err != nil { + return reconcile.Result{}, err + } + budget, err := intstr.GetScaledValueFromIntOrPercent(&budgetValue, int(liveDS.Status.DesiredNumberScheduled), true) + if err != nil || budget <= 0 { + return reconcile.Result{}, err + } + consumed := consumedFallbackBudget(liveDS, pods, currentRevision, time.Now()) + if consumed >= budget { + return reconcile.Result{}, nil + } + candidates := preparedHandoffCandidates(liveDS, pods, currentRevision) + for _, candidate := range candidates { + if !candidate.reserved { + base := candidate.replacement.DeepCopy() + patched := candidate.replacement.DeepCopy() + if patched.Annotations == nil { + patched.Annotations = map[string]string{} + } + patched.Annotations[resourceFallbackOldPodAnnotation] = string(candidate.old.UID) + if err := r.client.Patch(ctx, patched, client.MergeFrom(base)); err != nil { + return reconcile.Result{}, fmt.Errorf("reserve prepared Agent handoff for Pod %s/%s: %w", patched.Namespace, patched.Name, err) + } + candidate.replacement = patched + } + liveCandidate, err := r.revalidatePreparedHandoff(ctx, liveDS, candidate, currentRevision) + if err != nil { + return reconcile.Result{}, err + } + if liveCandidate == nil { + continue + } + withinBudget, err := fallbackBudgetWithinLimit(ctx, reader, liveDS, budgetValue, currentRevision) + if err != nil { + return reconcile.Result{}, err + } + if !withinBudget { + return reconcile.Result{RequeueAfter: time.Second}, nil + } + uid := liveCandidate.old.UID + if err := r.client.Delete(ctx, liveCandidate.old, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("delete old Agent Pod %s/%s for prepared handoff: %w", liveCandidate.old.Namespace, liveCandidate.old.Name, err) + } + if r.recorder != nil { + r.recorder.Eventf(ddai, corev1.EventTypeNormal, "AgentPreparedHandoff", "Deleted old Agent Pod %s on node %s after replacement %s reported Prepared", liveCandidate.old.Name, liveCandidate.nodeName, liveCandidate.replacement.Name) + } + return reconcile.Result{RequeueAfter: time.Second}, nil + } + return reconcile.Result{}, nil +} + +func preparedHandoffCandidates(ds *appsv1.DaemonSet, pods []corev1.Pod, currentRevision string) []preparedHandoffCandidate { + oldByNode := map[string]*corev1.Pod{} + for i := range pods { + pod := &pods[i] + if pod.Spec.NodeName != "" && pod.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] != currentRevision && podAvailable(pod, ds.Spec.MinReadySeconds, time.Now()) { + oldByNode[pod.Spec.NodeName] = pod + } + } + var candidates []preparedHandoffCandidate + for i := range pods { + pod := &pods[i] + if pod.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] != currentRevision || !podPreparedForHandoff(pod) || podAvailable(pod, ds.Spec.MinReadySeconds, time.Now()) { + continue + } + old := oldByNode[pod.Spec.NodeName] + if old == nil { + continue + } + reservation := pod.Annotations[resourceFallbackOldPodAnnotation] + if reservation != "" && reservation != string(old.UID) { + continue + } + candidates = append(candidates, preparedHandoffCandidate{replacement: pod, old: old, nodeName: pod.Spec.NodeName, reserved: reservation != ""}) + } + sort.Slice(candidates, func(i, j int) bool { return candidates[i].nodeName < candidates[j].nodeName }) + return candidates +} + +func podPreparedForHandoff(pod *corev1.Pod) bool { + if pod.DeletionTimestamp != nil || pod.Status.Phase != corev1.PodRunning || len(pod.Status.InitContainerStatuses) != 2 || len(pod.Status.ContainerStatuses) != len(preparedRolloutContainerNames) { + return false + } + for i := range pod.Status.InitContainerStatuses { + status := &pod.Status.InitContainerStatuses[i] + if status.State.Terminated == nil || status.State.Terminated.ExitCode != 0 { + return false + } + } + seen := map[string]bool{} + for i := range pod.Status.ContainerStatuses { + status := &pod.Status.ContainerStatuses[i] + if status.Name != string(apicommon.CoreAgentContainerName) && status.Name != string(apicommon.TraceAgentContainerName) || status.State.Running == nil || status.Started == nil || !*status.Started || status.RestartCount != 0 { + return false + } + seen[status.Name] = true + } + return seen[string(apicommon.CoreAgentContainerName)] && seen[string(apicommon.TraceAgentContainerName)] +} + +func (r *Reconciler) revalidatePreparedHandoff(ctx context.Context, expectedDS *appsv1.DaemonSet, candidate preparedHandoffCandidate, expectedRevision string) (*preparedHandoffCandidate, error) { + liveDS := &appsv1.DaemonSet{} + if err := r.apiReader.Get(ctx, client.ObjectKeyFromObject(expectedDS), liveDS); err != nil { + return nil, client.IgnoreNotFound(err) + } + if liveDS.UID != expectedDS.UID || liveDS.Generation != expectedDS.Generation || liveDS.Spec.Template.Annotations[preparedRolloutPhaseAnnotation] != preparedRolloutPhaseStandby { + return nil, nil + } + revision, err := currentDaemonSetRevision(ctx, r.apiReader, liveDS) + if err != nil || revision != expectedRevision { + return nil, err + } + replacement := &corev1.Pod{} + old := &corev1.Pod{} + if err := r.apiReader.Get(ctx, client.ObjectKeyFromObject(candidate.replacement), replacement); err != nil { + return nil, client.IgnoreNotFound(err) + } + if err := r.apiReader.Get(ctx, client.ObjectKeyFromObject(candidate.old), old); err != nil { + return nil, client.IgnoreNotFound(err) + } + if replacement.UID != candidate.replacement.UID || old.UID != candidate.old.UID || !controlledByUID(replacement, liveDS.UID) || !controlledByUID(old, liveDS.UID) || replacement.Spec.NodeName != candidate.nodeName || old.Spec.NodeName != candidate.nodeName { + return nil, nil + } + if replacement.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] != revision || old.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] == revision || replacement.Annotations[resourceFallbackOldPodAnnotation] != string(old.UID) || !podPreparedForHandoff(replacement) || !podAvailable(old, liveDS.Spec.MinReadySeconds, time.Now()) { + return nil, nil + } + return &preparedHandoffCandidate{replacement: replacement, old: old, nodeName: candidate.nodeName, reserved: true}, nil +} diff --git a/internal/controller/datadogagentinternal/prepared_rollout_test.go b/internal/controller/datadogagentinternal/prepared_rollout_test.go new file mode 100644 index 0000000000..ed5f74f632 --- /dev/null +++ b/internal/controller/datadogagentinternal/prepared_rollout_test.go @@ -0,0 +1,225 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// Copyright 2016-present Datadog, Inc. + +package datadogagentinternal + +import ( + "context" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPrepareAgentTemplate(t *testing.T) { + arm := preparedRolloutDaemonSet() + require.NoError(t, prepareAgentTemplate(arm, preparedRolloutPhaseArm)) + assert.Equal(t, preparedRolloutPhaseArm, arm.Spec.Template.Annotations[preparedRolloutPhaseAnnotation]) + assert.NotEmpty(t, arm.Spec.Template.Spec.Containers[0].Ports, "arming keeps scheduler-visible ports because it does not overlap Pods") + for _, container := range arm.Spec.Template.Spec.Containers { + require.NotNil(t, container.StartupProbe) + require.NotNil(t, container.StartupProbe.Exec) + require.NotNil(t, container.ReadinessProbe) + require.NotNil(t, container.ReadinessProbe.Exec) + assert.Equal(t, "true", envValue(container.Env, rolloutEnabledEnv)) + assert.Contains(t, envValue(container.Env, rolloutLockPathEnv), container.Name+".lock") + assert.Contains(t, envValue(container.Env, rolloutStatePathEnv), container.Name+".state") + } + assert.Equal(t, "trace-agent", arm.Spec.Template.Spec.Containers[1].Command[0]) + + standby := preparedRolloutDaemonSet() + require.NoError(t, prepareAgentTemplate(standby, preparedRolloutPhaseStandby)) + for _, container := range standby.Spec.Template.Spec.Containers { + assert.Empty(t, container.Ports) + } +} + +func TestConfigurePreparedRolloutArmsThenStaysInStandby(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, appsv1.AddToScheme(scheme)) + reader := fake.NewClientBuilder().WithScheme(scheme).Build() + r := &Reconciler{apiReader: reader} + ddai := &datadoghqv1alpha1.DatadogAgentInternal{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{preparedRolloutAnnotation: "true"}}} + one := intstr.FromInt(1) + + desired := preparedRolloutDaemonSet() + phase, err := r.configurePreparedRollout(context.Background(), ddai, desired, one) + require.NoError(t, err) + assert.Equal(t, preparedRolloutPhaseArm, phase) + assert.Equal(t, intstr.FromInt(0), *desired.Spec.UpdateStrategy.RollingUpdate.MaxSurge) + + live := desired.DeepCopy() + live.UID = types.UID("daemonset-uid") + live.Generation = 2 + live.Status = appsv1.DaemonSetStatus{ + ObservedGeneration: 2, + DesiredNumberScheduled: 2, + UpdatedNumberScheduled: 2, + NumberReady: 2, + NumberAvailable: 2, + } + require.NoError(t, reader.Create(context.Background(), live)) + + next := preparedRolloutDaemonSet() + phase, err = r.configurePreparedRollout(context.Background(), ddai, next, one) + require.NoError(t, err) + assert.Equal(t, preparedRolloutPhaseStandby, phase) + assert.Equal(t, intstr.FromInt(1), *next.Spec.UpdateStrategy.RollingUpdate.MaxSurge) + assert.Equal(t, intstr.FromInt(0), *next.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable) + + live.Spec = next.Spec + live.Status.NumberReady = 1 + live.Status.NumberAvailable = 1 + require.NoError(t, reader.Update(context.Background(), live)) + + afterFailure := preparedRolloutDaemonSet() + phase, err = r.configurePreparedRollout(context.Background(), ddai, afterFailure, one) + require.NoError(t, err) + assert.Equal(t, preparedRolloutPhaseStandby, phase, "a mixed or failed rollout must not oscillate back to arm") +} + +func TestConfigurePreparedRolloutRejectsUnsupportedContainerWithoutMutation(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, appsv1.AddToScheme(scheme)) + r := &Reconciler{apiReader: fake.NewClientBuilder().WithScheme(scheme).Build()} + ddai := &datadoghqv1alpha1.DatadogAgentInternal{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{preparedRolloutAnnotation: "true"}}} + desired := preparedRolloutDaemonSet() + desired.Spec.Template.Spec.Containers = append(desired.Spec.Template.Spec.Containers, corev1.Container{Name: "security-agent"}) + original := desired.DeepCopy() + + _, err := r.configurePreparedRollout(context.Background(), ddai, desired, intstr.FromInt(1)) + require.Error(t, err) + assert.Equal(t, original, desired) +} + +func TestPodPreparedForHandoff(t *testing.T) { + pod := preparedReplacementPod() + assert.True(t, podPreparedForHandoff(pod)) + pod.Status.ContainerStatuses[1].Started = ptr.To(false) + assert.False(t, podPreparedForHandoff(pod)) + pod.Status.ContainerStatuses[1].Started = ptr.To(true) + pod.Status.ContainerStatuses[1].RestartCount = 1 + assert.False(t, podPreparedForHandoff(pod)) +} + +func TestReconcilePreparedHandoffReservesThenDeletesOldPod(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, appsv1.AddToScheme(scheme)) + require.NoError(t, datadoghqv1alpha1.AddToScheme(scheme)) + + ddai := &datadoghqv1alpha1.DatadogAgentInternal{ + ObjectMeta: metav1.ObjectMeta{Name: "agent", Namespace: "datadog-agent", UID: "ddai-uid", Annotations: map[string]string{preparedRolloutAnnotation: "true"}}, + } + ds := preparedRolloutDaemonSet() + ds.UID = "daemonset-uid" + ds.Generation = 2 + ds.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: datadoghqv1alpha1.GroupVersion.String(), + Kind: "DatadogAgentInternal", + Name: ddai.Name, + UID: ddai.UID, + Controller: ptr.To(true), + }} + require.NoError(t, prepareAgentTemplate(ds, preparedRolloutPhaseStandby)) + require.True(t, configureResourceFallback(ds, intstr.FromInt(1))) + ds.Status = appsv1.DaemonSetStatus{ObservedGeneration: 2, DesiredNumberScheduled: 1} + + old := readyPod("old", "old-uid", "node-a", "old-revision", time.Now().Add(-time.Minute)) + old.Namespace = ds.Namespace + old.Labels["app"] = "agent" + old.OwnerReferences = []metav1.OwnerReference{daemonSetOwner(ds)} + replacement := preparedReplacementPod() + replacement.ObjectMeta = metav1.ObjectMeta{ + Name: "new", + Namespace: ds.Namespace, + UID: "new-uid", + Labels: map[string]string{ + "app": "agent", + appsv1.DefaultDaemonSetUniqueLabelKey: "new-revision", + }, + OwnerReferences: []metav1.OwnerReference{daemonSetOwner(ds)}, + } + replacement.Spec.NodeName = "node-a" + revision := controllerRevisionForTemplate(t, ds, "new-revision") + + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(ddai, ds, old, replacement, revision).Build() + r := &Reconciler{client: c, apiReader: c} + result, err := r.reconcilePreparedHandoff(context.Background(), ddai, ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.Equal(t, time.Second, result.RequeueAfter) + + err = c.Get(context.Background(), client.ObjectKeyFromObject(old), &corev1.Pod{}) + assert.True(t, apierrors.IsNotFound(err), "the exact old UID should be deleted after both replacement processes report Prepared") + updated := &corev1.Pod{} + require.NoError(t, c.Get(context.Background(), client.ObjectKeyFromObject(replacement), updated)) + assert.Equal(t, string(old.UID), updated.Annotations[resourceFallbackOldPodAnnotation]) +} + +func preparedReplacementPod() *corev1.Pod { + return &corev1.Pod{Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + InitContainerStatuses: []corev1.ContainerStatus{ + {Name: "init-volume", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 0}}}, + {Name: "init-config", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 0}}}, + }, + ContainerStatuses: []corev1.ContainerStatus{ + {Name: "agent", Started: ptr.To(true), State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}, + {Name: "trace-agent", Started: ptr.To(true), State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}, + }, + }} +} + +func preparedRolloutDaemonSet() *appsv1.DaemonSet { + one := intstr.FromInt(1) + return &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: "datadog-agent", Namespace: "datadog-agent"}, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "agent"}}, + UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ + Type: appsv1.RollingUpdateDaemonSetStrategyType, + RollingUpdate: &appsv1.RollingUpdateDaemonSet{ + MaxUnavailable: &one, + MaxSurge: &one, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "agent"}}, + Spec: corev1.PodSpec{ + HostNetwork: true, + InitContainers: []corev1.Container{ + {Name: "init-volume"}, + {Name: "init-config"}, + }, + Containers: []corev1.Container{ + {Name: "agent", Command: []string{"agent", "run"}, Ports: []corev1.ContainerPort{{ContainerPort: 8125}}}, + {Name: "trace-agent", Command: []string{"trace-loader", "/etc/datadog-agent/datadog.yaml", "trace-agent", "--config=/etc/datadog-agent/datadog.yaml"}, Ports: []corev1.ContainerPort{{ContainerPort: 8126}}}, + }, + }, + }, + }, + } +} + +func envValue(env []corev1.EnvVar, name string) string { + for i := range env { + if env[i].Name == name { + return env[i].Value + } + } + return "" +} diff --git a/internal/controller/datadogagentinternal/resource_fallback.go b/internal/controller/datadogagentinternal/resource_fallback.go index c9395c4d13..99f5a100bf 100644 --- a/internal/controller/datadogagentinternal/resource_fallback.go +++ b/internal/controller/datadogagentinternal/resource_fallback.go @@ -8,6 +8,7 @@ import ( "context" "encoding/json" "fmt" + "slices" "sort" "strconv" "strings" @@ -18,6 +19,8 @@ import ( apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" resourcehelper "k8s.io/component-helpers/resource" @@ -26,9 +29,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + datadoghqcommon "github.com/DataDog/datadog-operator/api/datadoghq/common" datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" datadoghqv2alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v2alpha1" componentagent "github.com/DataDog/datadog-operator/internal/controller/datadogagent/component/agent" + "github.com/DataDog/datadog-operator/pkg/constants" ) const ( @@ -65,6 +70,73 @@ func configureResourceFallback(ds *appsv1.DaemonSet, budget intstr.IntOrString) return false } +// prepareProfileAntiAffinityForSurge narrows the standard DAP anti-affinity so +// only old and new revisions of the same profile and DDA may overlap. Unknown +// or user-supplied anti-affinity fails closed. +func prepareProfileAntiAffinityForSurge(template *corev1.PodTemplateSpec) bool { + if template.Spec.Affinity == nil || template.Spec.Affinity.PodAntiAffinity == nil { + return true + } + if !apiequality.Semantic.DeepEqual(template.Spec.Affinity.PodAntiAffinity, broadAgentPodAntiAffinity()) { + return false + } + narrowed, ok := profileSurgePodAntiAffinity(template.Labels) + if !ok { + return false + } + template.Spec.Affinity.PodAntiAffinity = narrowed + return true +} + +func broadAgentPodAntiAffinity() *corev1.PodAntiAffinity { + return &corev1.PodAntiAffinity{RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: datadoghqcommon.AgentDeploymentComponentLabelKey, + Operator: metav1.LabelSelectorOpIn, + Values: []string{constants.DefaultAgentResourceSuffix}, + }}}, + TopologyKey: corev1.LabelHostname, + }}} +} + +func profileSurgePodAntiAffinity(podLabels map[string]string) (*corev1.PodAntiAffinity, bool) { + ddaName := podLabels[datadoghqcommon.AgentDeploymentNameLabelKey] + if ddaName == "" { + return nil, false + } + + profileRequirement := metav1.LabelSelectorRequirement{Key: constants.ProfileLabelKey} + if profileName := podLabels[constants.ProfileLabelKey]; profileName != "" { + profileRequirement.Operator = metav1.LabelSelectorOpNotIn + profileRequirement.Values = []string{profileName} + } else { + profileRequirement.Operator = metav1.LabelSelectorOpExists + } + componentRequirement := metav1.LabelSelectorRequirement{ + Key: datadoghqcommon.AgentDeploymentComponentLabelKey, + Operator: metav1.LabelSelectorOpIn, + Values: []string{constants.DefaultAgentResourceSuffix}, + } + + return &corev1.PodAntiAffinity{RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + componentRequirement, + {Key: datadoghqcommon.AgentDeploymentNameLabelKey, Operator: metav1.LabelSelectorOpIn, Values: []string{ddaName}}, + profileRequirement, + }}, + TopologyKey: corev1.LabelHostname, + }, + { + LabelSelector: &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + componentRequirement, + {Key: datadoghqcommon.AgentDeploymentNameLabelKey, Operator: metav1.LabelSelectorOpNotIn, Values: []string{ddaName}}, + }}, + TopologyKey: corev1.LabelHostname, + }, + }}, true +} + func positiveIntOrPercent(value *intstr.IntOrString) bool { if value == nil { return false @@ -116,7 +188,10 @@ func (r *Reconciler) reconcileResourceFallback(ctx context.Context, ddai *datado desired := int(ds.Status.DesiredNumberScheduled) budget, err := intstr.GetScaledValueFromIntOrPercent(&budgetValue, desired, true) - if err != nil || budget <= 0 { + if err != nil { + return reconcile.Result{}, fmt.Errorf("resolve Agent resource fallback budget: %w", err) + } + if budget <= 0 { return reconcile.Result{}, nil } @@ -356,7 +431,7 @@ func resourceOnlyUnschedulable(pod *corev1.Pod) (resourceShortage, bool) { } var shortage resourceShortage - for _, reason := range strings.Split(primary, ", ") { + for reason := range strings.SplitSeq(primary, ", ") { fields := strings.Fields(strings.ToLower(strings.TrimSpace(reason))) if len(fields) < 2 { return resourceShortage{}, false @@ -428,8 +503,13 @@ func resourceFallbackSchedulingShapeSafe(pod *corev1.Pod) bool { if pod.Spec.RuntimeClassName != nil || len(pod.Spec.TopologySpreadConstraints) > 0 { return false } - if pod.Spec.Affinity != nil && (pod.Spec.Affinity.PodAffinity != nil || pod.Spec.Affinity.PodAntiAffinity != nil) { - return false + if pod.Spec.Affinity != nil { + if pod.Spec.Affinity.PodAffinity != nil { + return false + } + if pod.Spec.Affinity.PodAntiAffinity != nil && !profileSurgePodAntiAffinitySafe(pod) { + return false + } } for _, container := range append(append([]corev1.Container{}, pod.Spec.InitContainers...), pod.Spec.Containers...) { for _, port := range container.Ports { @@ -447,6 +527,110 @@ func resourceFallbackSchedulingShapeSafe(pod *corev1.Pod) bool { return true } +// profileSurgePodAntiAffinitySafe recognizes the exact anti-affinity emitted +// for DatadogAgentProfiles. It excludes only other profiles, so deleting the +// old Pod of the same profile cannot reveal a hidden anti-affinity blocker. +func profileSurgePodAntiAffinitySafe(pod *corev1.Pod) bool { + expected, ok := profileSurgePodAntiAffinity(pod.Labels) + if !ok { + return false + } + return apiequality.Semantic.DeepEqual(pod.Spec.Affinity.PodAntiAffinity, expected) +} + +func profileSurgePodAntiAffinitySatisfied(pending *corev1.Pod, nodePods []corev1.Pod) (bool, error) { + if pending.Spec.Affinity != nil && pending.Spec.Affinity.PodAntiAffinity != nil { + if !profileSurgePodAntiAffinitySafe(pending) { + return false, nil + } + for _, term := range pending.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution { + selector, err := metav1.LabelSelectorAsSelector(term.LabelSelector) + if err != nil { + return false, fmt.Errorf("parse prepared Agent Pod anti-affinity: %w", err) + } + for i := range nodePods { + pod := &nodePods[i] + if pod.Namespace == pending.Namespace && selector.Matches(labels.Set(pod.Labels)) { + return false, nil + } + } + } + } + return true, nil +} + +// existingPodsAllowPendingByRequiredAntiAffinity checks the symmetric half of +// inter-pod anti-affinity: an already scheduled Pod can reject the pending +// replacement even when the replacement's own terms allow it. +func existingPodsAllowPendingByRequiredAntiAffinity(pending *corev1.Pod, existingPods []corev1.Pod, targetNodeName string) (bool, error) { + for i := range existingPods { + existing := &existingPods[i] + if existing.Spec.NodeName == "" { + continue + } + if existing.Spec.Affinity == nil || existing.Spec.Affinity.PodAntiAffinity == nil { + continue + } + for _, term := range existing.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution { + selector, err := podAffinityTermSelector(&term, existing.Labels) + if err != nil { + return false, fmt.Errorf("parse existing Pod %s/%s anti-affinity: %w", existing.Namespace, existing.Name, err) + } + if !selector.Matches(labels.Set(pending.Labels)) { + continue + } + if !affinityTermMaySelectNamespace(&term, existing.Namespace, pending.Namespace) { + continue + } + // Pods on the target node cover hostname topology. For wider + // topologies, conservatively reject because this lightweight check + // does not fetch every Node's topology labels. + if term.TopologyKey != corev1.LabelHostname || existing.Spec.NodeName == targetNodeName { + return false, nil + } + } + } + return true, nil +} + +func podAffinityTermSelector(term *corev1.PodAffinityTerm, sourceLabels map[string]string) (labels.Selector, error) { + selector, err := metav1.LabelSelectorAsSelector(term.LabelSelector) + if err != nil { + return nil, err + } + for _, key := range term.MatchLabelKeys { + if value, ok := sourceLabels[key]; ok { + requirement, err := labels.NewRequirement(key, selection.In, []string{value}) + if err != nil { + return nil, err + } + selector = selector.Add(*requirement) + } + } + for _, key := range term.MismatchLabelKeys { + if value, ok := sourceLabels[key]; ok { + requirement, err := labels.NewRequirement(key, selection.NotIn, []string{value}) + if err != nil { + return nil, err + } + selector = selector.Add(*requirement) + } + } + return selector, nil +} + +func affinityTermMaySelectNamespace(term *corev1.PodAffinityTerm, sourceNamespace, targetNamespace string) bool { + if slices.Contains(term.Namespaces, targetNamespace) { + return true + } + // Namespace labels are intentionally not part of the fallback cache. Fail + // closed when a namespace selector could include the pending Pod. + if term.NamespaceSelector != nil { + return true + } + return len(term.Namespaces) == 0 && sourceNamespace == targetNamespace +} + func consumedFallbackBudget(ds *appsv1.DaemonSet, pods []corev1.Pod, currentRevision string, now time.Time) int { availableByNode := map[string]bool{} knownNodes := map[string]bool{} @@ -525,12 +709,12 @@ func (r *Reconciler) revalidateFallbackCandidate(ctx context.Context, reader cli } pending := &corev1.Pod{} - if err := reader.Get(ctx, client.ObjectKeyFromObject(candidate.pending), pending); err != nil { - return nil, client.IgnoreNotFound(err) + if getErr := reader.Get(ctx, client.ObjectKeyFromObject(candidate.pending), pending); getErr != nil { + return nil, client.IgnoreNotFound(getErr) } old := &corev1.Pod{} - if err := reader.Get(ctx, client.ObjectKeyFromObject(candidate.old), old); err != nil { - return nil, client.IgnoreNotFound(err) + if getErr := reader.Get(ctx, client.ObjectKeyFromObject(candidate.old), old); getErr != nil { + return nil, client.IgnoreNotFound(getErr) } if !controlledByUID(pending, liveDS.UID) || !controlledByUID(old, liveDS.UID) || pending.UID != candidate.pending.UID || old.UID != candidate.old.UID { return nil, nil @@ -546,19 +730,40 @@ func (r *Reconciler) revalidateFallbackCandidate(ctx context.Context, reader cli } node := &corev1.Node{} - if err := reader.Get(ctx, client.ObjectKey{Name: nodeName}, node); err != nil { - return nil, client.IgnoreNotFound(err) + if getErr := reader.Get(ctx, client.ObjectKey{Name: nodeName}, node); getErr != nil { + return nil, client.IgnoreNotFound(getErr) } if !nodeReadyForResourceFallback(node) { return nil, nil } + if !toleratesBlockingNodeTaints(pending.Spec.Tolerations, node.Spec.Taints) { + return nil, nil + } matches, err := nodeaffinity.GetRequiredNodeAffinity(pending).Match(node) if err != nil || !matches { return nil, err } nodePods := &corev1.PodList{} - if err := reader.List(ctx, nodePods, client.MatchingFields{apiPodNodeNameField: nodeName}); err != nil { - return nil, fmt.Errorf("list Pods on node %s for Agent resource fallback: %w", nodeName, err) + if listErr := reader.List(ctx, nodePods, client.MatchingFields{apiPodNodeNameField: nodeName}); listErr != nil { + return nil, fmt.Errorf("list Pods on node %s for Agent resource fallback: %w", nodeName, listErr) + } + affinitySatisfied, err := profileSurgePodAntiAffinitySatisfied(pending, nodePods.Items) + if err != nil { + return nil, err + } + if !affinitySatisfied { + return nil, nil + } + clusterPods := &corev1.PodList{} + if listErr := reader.List(ctx, clusterPods); listErr != nil { + return nil, fmt.Errorf("list cluster Pods for Agent anti-affinity fallback safety: %w", listErr) + } + existingAffinitySatisfied, err := existingPodsAllowPendingByRequiredAntiAffinity(pending, clusterPods.Items, nodeName) + if err != nil { + return nil, err + } + if !existingAffinitySatisfied { + return nil, nil } if !resourceFitAfterOldPodRemoval(node, nodePods.Items, pending, old, shortage) { return nil, nil @@ -567,6 +772,39 @@ func (r *Reconciler) revalidateFallbackCandidate(ctx context.Context, reader cli return &fallbackCandidate{pending: pending, old: old, nodeName: nodeName, shortage: shortage, reserved: reservation != ""}, nil } +func toleratesBlockingNodeTaints(tolerations []corev1.Toleration, taints []corev1.Taint) bool { + for i := range taints { + taint := &taints[i] + if taint.Effect != corev1.TaintEffectNoSchedule && taint.Effect != corev1.TaintEffectNoExecute { + continue + } + tolerated := false + for j := range tolerations { + toleration := &tolerations[j] + if toleration.Effect != "" && toleration.Effect != taint.Effect { + continue + } + operator := toleration.Operator + if operator == "" { + operator = corev1.TolerationOpEqual + } + switch operator { + case corev1.TolerationOpExists: + tolerated = toleration.Key == "" || toleration.Key == taint.Key + case corev1.TolerationOpEqual: + tolerated = toleration.Key == taint.Key && toleration.Value == taint.Value + } + if tolerated { + break + } + } + if !tolerated { + return false + } + } + return true +} + func nodeReadyForResourceFallback(node *corev1.Node) bool { if node.Spec.Unschedulable || node.DeletionTimestamp != nil { return false diff --git a/internal/controller/datadogagentinternal/resource_fallback_test.go b/internal/controller/datadogagentinternal/resource_fallback_test.go index f1b2b5e277..2727d8364b 100644 --- a/internal/controller/datadogagentinternal/resource_fallback_test.go +++ b/internal/controller/datadogagentinternal/resource_fallback_test.go @@ -7,6 +7,7 @@ package datadogagentinternal import ( "context" "encoding/json" + "maps" "testing" "time" @@ -15,6 +16,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" @@ -22,7 +24,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + datadoghqcommon "github.com/DataDog/datadog-operator/api/datadoghq/common" datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" + "github.com/DataDog/datadog-operator/pkg/constants" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -130,6 +134,154 @@ func TestTargetNodeFromDaemonSetAffinity(t *testing.T) { assert.False(t, ok) } +func TestResourceFallbackSchedulingShapeAllowsOnlyProfileSurgeAntiAffinity(t *testing.T) { + namedLabels := map[string]string{ + datadoghqcommon.AgentDeploymentNameLabelKey: "datadog-agent", + constants.ProfileLabelKey: "linux", + } + namedAntiAffinity, ok := profileSurgePodAntiAffinity(namedLabels) + require.True(t, ok) + named := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Labels: namedLabels}, Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{PodAntiAffinity: namedAntiAffinity}, + Containers: []corev1.Container{{Name: "agent"}}, + }} + assert.True(t, resourceFallbackSchedulingShapeSafe(named)) + + defaultProfile := named.DeepCopy() + defaultProfile.Labels = map[string]string{datadoghqcommon.AgentDeploymentNameLabelKey: "datadog-agent"} + defaultProfile.Spec.Affinity.PodAntiAffinity, ok = profileSurgePodAntiAffinity(defaultProfile.Labels) + require.True(t, ok) + assert.True(t, resourceFallbackSchedulingShapeSafe(defaultProfile)) + + wrongProfile := named.DeepCopy() + wrongProfile.Spec.Affinity.PodAntiAffinity, ok = profileSurgePodAntiAffinity(map[string]string{ + datadoghqcommon.AgentDeploymentNameLabelKey: "datadog-agent", + constants.ProfileLabelKey: "gpu", + }) + require.True(t, ok) + assert.False(t, resourceFallbackSchedulingShapeSafe(wrongProfile)) + + custom := named.DeepCopy() + custom.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].TopologyKey = "topology.kubernetes.io/zone" + assert.False(t, resourceFallbackSchedulingShapeSafe(custom)) +} + +func TestProfileSurgePodAntiAffinityIdentity(t *testing.T) { + incomingLabels := map[string]string{ + datadoghqcommon.AgentDeploymentNameLabelKey: "datadog-agent", + constants.ProfileLabelKey: "linux", + } + antiAffinity, ok := profileSurgePodAntiAffinity(incomingLabels) + require.True(t, ok) + + conflicts := func(existingLabels map[string]string) bool { + for _, term := range antiAffinity.RequiredDuringSchedulingIgnoredDuringExecution { + selector, err := metav1.LabelSelectorAsSelector(term.LabelSelector) + require.NoError(t, err) + if selector.Matches(labels.Set(existingLabels)) { + return true + } + } + return false + } + + assert.False(t, conflicts(map[string]string{ + datadoghqcommon.AgentDeploymentComponentLabelKey: constants.DefaultAgentResourceSuffix, + datadoghqcommon.AgentDeploymentNameLabelKey: "datadog-agent", + constants.ProfileLabelKey: "linux", + }), "old and new revisions of the same DDA profile may overlap") + assert.True(t, conflicts(map[string]string{ + datadoghqcommon.AgentDeploymentComponentLabelKey: constants.DefaultAgentResourceSuffix, + datadoghqcommon.AgentDeploymentNameLabelKey: "datadog-agent", + constants.ProfileLabelKey: "gpu", + }), "another profile of the same DDA must remain excluded") + assert.True(t, conflicts(map[string]string{ + datadoghqcommon.AgentDeploymentComponentLabelKey: constants.DefaultAgentResourceSuffix, + datadoghqcommon.AgentDeploymentNameLabelKey: "other-datadog-agent", + constants.ProfileLabelKey: "linux", + }), "the same profile name from another DDA must remain excluded") +} + +func TestProfileSurgePodAntiAffinitySatisfiedOnTargetNode(t *testing.T) { + pendingLabels := map[string]string{ + datadoghqcommon.AgentDeploymentNameLabelKey: "datadog-agent", + datadoghqcommon.AgentDeploymentComponentLabelKey: constants.DefaultAgentResourceSuffix, + constants.ProfileLabelKey: "linux", + } + antiAffinity, ok := profileSurgePodAntiAffinity(pendingLabels) + require.True(t, ok) + pending := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "datadog", Labels: pendingLabels}, Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{PodAntiAffinity: antiAffinity}, + }} + sameProfile := corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "datadog", Labels: maps.Clone(pendingLabels)}} + + satisfied, err := profileSurgePodAntiAffinitySatisfied(pending, []corev1.Pod{sameProfile}) + require.NoError(t, err) + assert.True(t, satisfied) + + otherProfile := sameProfile.DeepCopy() + otherProfile.Labels[constants.ProfileLabelKey] = "gpu" + satisfied, err = profileSurgePodAntiAffinitySatisfied(pending, []corev1.Pod{sameProfile, *otherProfile}) + require.NoError(t, err) + assert.False(t, satisfied, "a masked different-profile blocker must prevent old Pod deletion") + + otherProfile.Namespace = "another-namespace" + satisfied, err = profileSurgePodAntiAffinitySatisfied(pending, []corev1.Pod{sameProfile, *otherProfile}) + require.NoError(t, err) + assert.True(t, satisfied, "Pod anti-affinity without explicit namespaces is namespace-scoped") +} + +func TestExistingPodRequiredAntiAffinityCanRejectReplacement(t *testing.T) { + pending := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "datadog", Labels: map[string]string{ + datadoghqcommon.AgentDeploymentComponentLabelKey: constants.DefaultAgentResourceSuffix, + "rollout": "new", + }}} + existing := corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "datadog", Name: "peer", Labels: map[string]string{"rollout": "old"}}, Spec: corev1.PodSpec{ + NodeName: "node-a", + Affinity: &corev1.Affinity{PodAntiAffinity: &corev1.PodAntiAffinity{RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ + datadoghqcommon.AgentDeploymentComponentLabelKey: constants.DefaultAgentResourceSuffix, + }}, + TopologyKey: corev1.LabelHostname, + }}}}, + }} + + allowed, err := existingPodsAllowPendingByRequiredAntiAffinity(pending, []corev1.Pod{existing}, "node-a") + require.NoError(t, err) + assert.False(t, allowed, "an existing Pod's required anti-affinity must block fallback deletion") + + existing.Namespace = "another-namespace" + allowed, err = existingPodsAllowPendingByRequiredAntiAffinity(pending, []corev1.Pod{existing}, "node-a") + require.NoError(t, err) + assert.True(t, allowed) + + emptyNamespaceSelector := &metav1.LabelSelector{} + existing.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].NamespaceSelector = emptyNamespaceSelector + allowed, err = existingPodsAllowPendingByRequiredAntiAffinity(pending, []corev1.Pod{existing}, "node-a") + require.NoError(t, err) + assert.False(t, allowed, "namespace selectors fail closed because namespace labels are not cached") + + existing.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector = &metav1.LabelSelector{MatchLabels: map[string]string{"rollout": "old"}} + existing.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].NamespaceSelector = nil + existing.Namespace = "datadog" + allowed, err = existingPodsAllowPendingByRequiredAntiAffinity(pending, []corev1.Pod{existing}, "node-a") + require.NoError(t, err) + assert.True(t, allowed, "non-matching selectors do not block the replacement") + + existing.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector = &metav1.LabelSelector{MatchLabels: map[string]string{ + datadoghqcommon.AgentDeploymentComponentLabelKey: constants.DefaultAgentResourceSuffix, + }} + existing.Spec.NodeName = "node-b" + allowed, err = existingPodsAllowPendingByRequiredAntiAffinity(pending, []corev1.Pod{existing}, "node-a") + require.NoError(t, err) + assert.True(t, allowed, "hostname anti-affinity on another node does not block") + + existing.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].TopologyKey = "topology.kubernetes.io/zone" + allowed, err = existingPodsAllowPendingByRequiredAntiAffinity(pending, []corev1.Pod{existing}, "node-a") + require.NoError(t, err) + assert.False(t, allowed, "wider topology terms fail closed without loading every Node's topology labels") +} + func TestResourceFitAfterOldPodRemoval(t *testing.T) { node := &corev1.Node{Status: corev1.NodeStatus{Allocatable: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1500m"), @@ -222,6 +374,12 @@ func TestReconcileResourceFallbackKeepsOldPodForHiddenSchedulerConstraints(t *te pod.Spec.Affinity.PodAffinity = &corev1.PodAffinity{RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{TopologyKey: "kubernetes.io/hostname", LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "peer"}}}}} }, }, + { + name: "unrecognized pod anti affinity", + mutate: func(pod *corev1.Pod) { + pod.Spec.Affinity.PodAntiAffinity = &corev1.PodAntiAffinity{RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{TopologyKey: "topology.kubernetes.io/zone"}}} + }, + }, { name: "declared host port", mutate: func(pod *corev1.Pod) { @@ -291,6 +449,23 @@ func TestReconcileResourceFallbackRejectsForeignDaemonSet(t *testing.T) { require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{}), "foreign DaemonSet Pods must never be deleted") } +func TestToleratesBlockingNodeTaints(t *testing.T) { + taints := []corev1.Taint{ + {Key: "dedicated", Value: "agents", Effect: corev1.TaintEffectNoSchedule}, + {Key: "draining", Effect: corev1.TaintEffectNoExecute}, + {Key: "soft", Effect: corev1.TaintEffectPreferNoSchedule}, + } + assert.False(t, toleratesBlockingNodeTaints(nil, taints)) + assert.False(t, toleratesBlockingNodeTaints([]corev1.Toleration{ + {Key: "dedicated", Value: "agents", Operator: corev1.TolerationOpEqual, Effect: corev1.TaintEffectNoSchedule}, + }, taints)) + assert.True(t, toleratesBlockingNodeTaints([]corev1.Toleration{ + {Key: "dedicated", Value: "agents", Operator: corev1.TolerationOpEqual, Effect: corev1.TaintEffectNoSchedule}, + {Key: "draining", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoExecute}, + }, taints)) + assert.True(t, toleratesBlockingNodeTaints([]corev1.Toleration{{Operator: corev1.TolerationOpExists}}, taints)) +} + type fallbackTestFixture struct { client client.Client reconciler *Reconciler diff --git a/internal/controller/datadogagentinternal_controller.go b/internal/controller/datadogagentinternal_controller.go index 079f0bb764..ace8799025 100644 --- a/internal/controller/datadogagentinternal_controller.go +++ b/internal/controller/datadogagentinternal_controller.go @@ -13,6 +13,7 @@ import ( corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -160,13 +161,35 @@ func resourceFallbackPodPredicate() predicate.Predicate { if !oldOK || !newOK { return false } - return resourceFallbackConditionChanged(oldPod, newPod, corev1.PodScheduled) || resourceFallbackConditionChanged(oldPod, newPod, corev1.PodReady) + return resourceFallbackConditionChanged(oldPod, newPod, corev1.PodScheduled) || + resourceFallbackConditionChanged(oldPod, newPod, corev1.PodReady) || + containerRolloutStatusChanged(oldPod.Status.InitContainerStatuses, newPod.Status.InitContainerStatuses) || + containerRolloutStatusChanged(oldPod.Status.ContainerStatuses, newPod.Status.ContainerStatuses) }, DeleteFunc: func(event.DeleteEvent) bool { return true }, GenericFunc: func(event.GenericEvent) bool { return false }, } } +func containerRolloutStatusChanged(oldStatuses, newStatuses []corev1.ContainerStatus) bool { + if len(oldStatuses) != len(newStatuses) { + return true + } + oldByName := make(map[string]corev1.ContainerStatus, len(oldStatuses)) + for i := range oldStatuses { + oldByName[oldStatuses[i].Name] = oldStatuses[i] + } + for i := range newStatuses { + old, found := oldByName[newStatuses[i].Name] + if !found || !apiequality.Semantic.DeepEqual(old.Started, newStatuses[i].Started) || old.RestartCount != newStatuses[i].RestartCount || + (old.State.Running == nil) != (newStatuses[i].State.Running == nil) || + (old.State.Terminated == nil) != (newStatuses[i].State.Terminated == nil) { + return true + } + } + return false +} + func resourceFallbackConditionChanged(oldPod, newPod *corev1.Pod, conditionType corev1.PodConditionType) bool { oldCondition := podCondition(oldPod, conditionType) newCondition := podCondition(newPod, conditionType) diff --git a/internal/controller/datadogagentinternal_controller_test.go b/internal/controller/datadogagentinternal_controller_test.go index d51af40e36..7ddd48ab2b 100644 --- a/internal/controller/datadogagentinternal_controller_test.go +++ b/internal/controller/datadogagentinternal_controller_test.go @@ -34,6 +34,12 @@ func TestResourceFallbackPodPredicate(t *testing.T) { readyPod := newPod.DeepCopy() readyPod.Status.Conditions = append(readyPod.Status.Conditions, corev1.PodCondition{Type: corev1.PodReady, Status: corev1.ConditionTrue}) assert.True(t, predicate.Update(event.UpdateEvent{ObjectOld: newPod, ObjectNew: readyPod}), "PodReady transitions must release fallback reservations promptly") + + waiting := readyPod.DeepCopy() + waiting.Status.ContainerStatuses = []corev1.ContainerStatus{{Name: "agent", Started: ptr.To(false), State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}} + prepared := waiting.DeepCopy() + prepared.Status.ContainerStatuses[0].Started = ptr.To(true) + assert.True(t, predicate.Update(event.UpdateEvent{ObjectOld: waiting, ObjectNew: prepared}), "startup-probe Prepared transitions must enqueue the handoff controller") } func TestEnqueueDatadogAgentInternalForPodFollowsDaemonSetOwner(t *testing.T) { diff --git a/internal/controller/testutils/renderer/render_e2e_test.go b/internal/controller/testutils/renderer/render_e2e_test.go index 89dbe07234..14cd89c98b 100644 --- a/internal/controller/testutils/renderer/render_e2e_test.go +++ b/internal/controller/testutils/renderer/render_e2e_test.go @@ -9,6 +9,12 @@ import ( "strings" "testing" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + + common "github.com/DataDog/datadog-operator/api/datadoghq/common" + datadoghqv2alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v2alpha1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/runtime" @@ -159,6 +165,125 @@ func TestRender_AppArmorProfileVersionGate(t *testing.T) { } } +func TestRender_PreparedRolloutArmsBeforeSurge(t *testing.T) { + renderAgentDaemonSet := func(t *testing.T, prepared bool) *appsv1.DaemonSet { + t.Helper() + dda, err := LoadDDA("testdata/minimal-dda.yaml") + require.NoError(t, err) + dda.Spec.Features = preparedRolloutTestFeatures() + + override := &datadoghqv2alpha1.DatadogAgentComponentOverride{HostNetwork: ptr.To(true)} + override.UpdateStrategy = &common.UpdateStrategy{ + Type: string(appsv1.RollingUpdateDaemonSetStrategyType), + RollingUpdate: &common.RollingUpdate{ + MaxUnavailable: ptr.To(intstr.FromInt(1)), + MaxSurge: ptr.To(intstr.FromInt(1)), + }, + } + if prepared { + dda.Annotations = map[string]string{"experimental.agent.datadoghq.com/host-network-surge-prepared": "true"} + } + dda.Spec.Override = map[datadoghqv2alpha1.ComponentName]*datadoghqv2alpha1.DatadogAgentComponentOverride{ + datadoghqv2alpha1.NodeAgentComponentName: override, + } + + objects, _, err := Render(Options{DDA: dda}) + require.NoError(t, err) + for _, object := range objects { + if ds, ok := object.(*appsv1.DaemonSet); ok { + return ds + } + } + t.Fatal("render produced no Agent DaemonSet") + return nil + } + + baseline := renderAgentDaemonSet(t, false) + require.True(t, baseline.Spec.Template.Spec.HostNetwork) + require.NotNil(t, baseline.Spec.UpdateStrategy.RollingUpdate) + assert.Equal(t, intstr.FromInt(1), *baseline.Spec.UpdateStrategy.RollingUpdate.MaxSurge) + assert.Equal(t, intstr.FromInt(1), *baseline.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable, + "ordinary native surge must remain unchanged when prepared rollout is disabled") + baselinePortCount := 0 + for _, container := range baseline.Spec.Template.Spec.Containers { + baselinePortCount += len(container.Ports) + } + require.Positive(t, baselinePortCount, "the host-network baseline must exercise Kubernetes's implicit hostPort defaulting") + + armed := renderAgentDaemonSet(t, true) + require.True(t, armed.Spec.Template.Spec.HostNetwork) + require.NotNil(t, armed.Spec.UpdateStrategy.RollingUpdate) + assert.Equal(t, intstr.FromInt(0), *armed.Spec.UpdateStrategy.RollingUpdate.MaxSurge) + assert.Equal(t, intstr.FromInt(1), *armed.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable) + assert.Equal(t, "arm", armed.Spec.Template.Annotations["experimental.agent.datadoghq.com/prepared-rollout-phase"]) + armedPortCount := 0 + for _, container := range armed.Spec.Template.Spec.Containers { + armedPortCount += len(container.Ports) + require.NotNil(t, container.StartupProbe) + require.NotNil(t, container.StartupProbe.Exec) + require.NotNil(t, container.ReadinessProbe) + require.NotNil(t, container.ReadinessProbe.Exec) + if container.Name == "trace-agent" { + assert.Equal(t, "trace-agent", container.Command[0], "prepared mode must bypass trace-loader") + } + } + assert.Equal(t, baselinePortCount, armedPortCount, "arming is a conventional rollout and keeps port declarations") +} + +func TestRender_PreparedHostNetworkSurgeWithProfiles(t *testing.T) { + dda, err := LoadDDA("testdata/minimal-dda.yaml") + require.NoError(t, err) + dda.Spec.Features = preparedRolloutTestFeatures() + dda.Annotations = map[string]string{"experimental.agent.datadoghq.com/host-network-surge-prepared": "true"} + + surgeOverride := func() *datadoghqv2alpha1.DatadogAgentComponentOverride { + return &datadoghqv2alpha1.DatadogAgentComponentOverride{ + HostNetwork: ptr.To(true), + UpdateStrategy: &common.UpdateStrategy{ + Type: string(appsv1.RollingUpdateDaemonSetStrategyType), + RollingUpdate: &common.RollingUpdate{ + MaxUnavailable: ptr.To(intstr.FromInt(1)), + MaxSurge: ptr.To(intstr.FromInt(1)), + }, + }, + } + } + dda.Spec.Override = map[datadoghqv2alpha1.ComponentName]*datadoghqv2alpha1.DatadogAgentComponentOverride{ + datadoghqv2alpha1.NodeAgentComponentName: surgeOverride(), + } + + daps, err := LoadDAPs([]string{"testdata/linux-profile.yaml", "testdata/gpu-profile.yaml"}) + require.NoError(t, err) + + objects, _, err := Render(Options{DDA: dda, DAPs: daps, ProfileEnabled: true}) + require.NoError(t, err) + daemonSets := 0 + for _, object := range objects { + ds, ok := object.(*appsv1.DaemonSet) + if !ok { + continue + } + daemonSets++ + require.True(t, ds.Spec.Template.Spec.HostNetwork) + assert.Equal(t, "arm", ds.Spec.Template.Annotations["experimental.agent.datadoghq.com/prepared-rollout-phase"]) + require.NotNil(t, ds.Spec.Template.Spec.Affinity) + require.NotNil(t, ds.Spec.Template.Spec.Affinity.PodAntiAffinity) + assert.Len(t, ds.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, 2, + "DaemonSet %s must scope overlap to the same DDA and profile", ds.Name) + } + assert.Equal(t, 3, daemonSets) +} + +func preparedRolloutTestFeatures() *datadoghqv2alpha1.DatadogFeatures { + return &datadoghqv2alpha1.DatadogFeatures{ + APM: &datadoghqv2alpha1.APMFeatureConfig{Enabled: ptr.To(true)}, + LiveProcessCollection: &datadoghqv2alpha1.LiveProcessCollectionFeatureConfig{Enabled: ptr.To(false)}, + LiveContainerCollection: &datadoghqv2alpha1.LiveContainerCollectionFeatureConfig{Enabled: ptr.To(false)}, + ProcessDiscovery: &datadoghqv2alpha1.ProcessDiscoveryFeatureConfig{Enabled: ptr.To(false)}, + ServiceDiscovery: &datadoghqv2alpha1.ServiceDiscoveryFeatureConfig{Enabled: ptr.To(false)}, + } +} + // kindSequence extracts the ordered list of "kind: X" values from serialized YAML. func kindSequence(yaml string) []string { var kinds []string diff --git a/pkg/config/config.go b/pkg/config/config.go index 413dade901..882e4b96e7 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -172,6 +172,9 @@ func CacheOptions(logger logr.Logger, opts WatchOptions) cache.Options { }, } newPod.Status.Conditions = pod.Status.Conditions + newPod.Status.Phase = pod.Status.Phase + newPod.Status.InitContainerStatuses = pod.Status.InitContainerStatuses + newPod.Status.ContainerStatuses = pod.Status.ContainerStatuses // The untaint controller also needs Pod.Status.StartTime for its // readiness-timeout clock. diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index a4a30ffaed..2a5bba8fbc 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -6,8 +6,11 @@ import ( "testing" "golang.org/x/exp/maps" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -292,3 +295,35 @@ func verifyResourceNamespace(t *testing.T, resource client.Object, wantConfig ob } } } + +func TestAgentPodCacheTransformPreservesPreparedRolloutStatus(t *testing.T) { + t.Setenv(AgentWatchNamespaceEnvVar, "datadog-agent") + options := CacheOptions(logf.Log.WithName(t.Name()), WatchOptions{DatadogAgentEnabled: true}) + podConfig, found := options.ByObject[podObj] + require.True(t, found) + require.NotNil(t, podConfig.Transform) + + started := true + input := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "agent", Namespace: "datadog-agent"}, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + InitContainerStatuses: []corev1.ContainerStatus{{ + Name: "init-config", + State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 0}}, + }}, + ContainerStatuses: []corev1.ContainerStatus{{ + Name: "agent", + Started: &started, + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }}, + }, + } + transformedObject, err := podConfig.Transform(input) + require.NoError(t, err) + transformed := transformedObject.(*corev1.Pod) + assert.Equal(t, corev1.PodRunning, transformed.Status.Phase) + require.Len(t, transformed.Status.InitContainerStatuses, 1) + require.Len(t, transformed.Status.ContainerStatuses, 1) + assert.True(t, *transformed.Status.ContainerStatuses[0].Started) +} From f7afeb57cea248099fa720e19a2611367e41755e Mon Sep 17 00:00:00 2001 From: "ali.benabdallah" Date: Wed, 22 Jul 2026 18:32:10 +0200 Subject: [PATCH 03/16] Preserve active Agent health probes --- docs/agent_host_network_surge_poc.md | 5 ++- docs/agent_zero_gap_rollouts_rfc.md | 5 ++- docs/agent_zero_gap_rollouts_rfc_appendix.md | 8 ++-- .../datadogagentinternal/prepared_rollout.go | 37 ++++++++++++++++++- .../prepared_rollout_test.go | 5 +++ 5 files changed, 51 insertions(+), 9 deletions(-) diff --git a/docs/agent_host_network_surge_poc.md b/docs/agent_host_network_surge_poc.md index 16201e2000..8c93dd31e8 100644 --- a/docs/agent_host_network_surge_poc.md +++ b/docs/agent_host_network_surge_poc.md @@ -47,8 +47,9 @@ The first pilot fails closed unless the rendered Pod has exactly: The Operator injects per-component node lock paths and Pod-private state paths, bypasses `trace-loader`, and replaces network probes with state-file exec -probes. Startup/liveness accept Prepared or Active; readiness accepts only -Active. Once both containers have `Started=true`, the Operator annotates the +probes. Startup accepts Prepared, waiting liveness accepts Prepared, and +post-activation liveness/readiness delegate to `agent health` or the local APM +listener. Once both containers have `Started=true`, the Operator annotates the replacement with the old Pod UID and deletes that exact UID within the `maxUnavailable` budget. The node locks keep the new processes asleep until the old processes finish stopping. diff --git a/docs/agent_zero_gap_rollouts_rfc.md b/docs/agent_zero_gap_rollouts_rfc.md index d487003085..e76734e62e 100644 --- a/docs/agent_zero_gap_rollouts_rfc.md +++ b/docs/agent_zero_gap_rollouts_rfc.md @@ -56,8 +56,9 @@ itself is not the UDS conflict—the bind/unlink behavior is. from the user's existing `maxUnavailable` budget. 3. A replacement constructs the real process graph, writes `prepared` to its Pod-private state file, then waits on a per-component advisory `flock` in a - stable node hostPath. Startup and liveness exec probes accept Prepared; - readiness accepts only Active. + stable node hostPath. Startup accepts Prepared. While waiting, liveness + accepts Prepared; after activation, liveness and readiness delegate to the + component's real health mechanism so the state marker cannot mask a failure. 4. When every supported replacement container is Running and `ContainerStatus.Started=true`, the Operator persists a token on that Pod and UID-precondition deletes the old Pod. The old processes release their locks diff --git a/docs/agent_zero_gap_rollouts_rfc_appendix.md b/docs/agent_zero_gap_rollouts_rfc_appendix.md index fc8ac3973b..ecb96a1b9e 100644 --- a/docs/agent_zero_gap_rollouts_rfc_appendix.md +++ b/docs/agent_zero_gap_rollouts_rfc_appendix.md @@ -353,9 +353,11 @@ parsing, and safe initialization costs before the old process exits. Every supported regular container gets a startup exec probe that accepts `prepared`, `activating`, or `active`. Kubelet then records `ContainerStatus.Started=true` for that exact container; a restart resets it. -Liveness accepts the same states so waiting is indefinite. Readiness accepts -only `active`, so a standby replacement is never a Service endpoint and never -looks healthy merely because the old host-network listener answers. +Liveness accepts the waiting states so waiting is indefinite. Once `active`, +core probes delegate to `agent health` and trace probes connect to the +container-local APM listener. Readiness requires Active and the same real +health check, so a standby replacement is never a Service endpoint, cannot hit +the old host-network listener, and cannot remain healthy on a stale state file. The Operator maintains a handoff budget derived from the user's existing `maxUnavailable` policy. When all expected containers are Running+Started, all diff --git a/internal/controller/datadogagentinternal/prepared_rollout.go b/internal/controller/datadogagentinternal/prepared_rollout.go index fc0773ebc1..bd72dd9fb2 100644 --- a/internal/controller/datadogagentinternal/prepared_rollout.go +++ b/internal/controller/datadogagentinternal/prepared_rollout.go @@ -264,6 +264,11 @@ func addPreparedRolloutVolumes(spec *corev1.PodSpec) error { func configurePreparedContainer(container *corev1.Container) { lockPath := preparedRolloutLockDir + "/" + container.Name + ".lock" statePath := preparedRolloutStateDir + "/" + container.Name + ".state" + originalLiveness := container.LivenessProbe.DeepCopy() + originalReadiness := container.ReadinessProbe.DeepCopy() + if originalReadiness == nil { + originalReadiness = originalLiveness + } setContainerEnv(container, rolloutEnabledEnv, "true") setContainerEnv(container, rolloutLockPathEnv, lockPath) setContainerEnv(container, rolloutStatePathEnv, statePath) @@ -272,8 +277,8 @@ func configurePreparedContainer(container *corev1.Container) { corev1.VolumeMount{Name: preparedRolloutStateVolume, MountPath: preparedRolloutStateDir}, ) container.StartupProbe = rolloutStateProbe(statePath, "prepared|activating|active", 1, 300) - container.LivenessProbe = rolloutStateProbe(statePath, "prepared|activating|active", 10, 3) - container.ReadinessProbe = rolloutStateProbe(statePath, "active", 1, 3) + container.LivenessProbe = rolloutHealthProbe(container.Name, statePath, true, originalLiveness) + container.ReadinessProbe = rolloutHealthProbe(container.Name, statePath, false, originalReadiness) } func setContainerEnv(container *corev1.Container, name, value string) { @@ -296,6 +301,34 @@ func rolloutStateProbe(path, accepted string, period, failures int32) *corev1.Pr } } +// rolloutHealthProbe accepts a sleeping Prepared process for liveness, but +// delegates to the component's real health mechanism after activation. This +// avoids both overlap false positives and permanently replacing Agent health +// with a state marker. +func rolloutHealthProbe(containerName, statePath string, allowWaiting bool, base *corev1.Probe) *corev1.Probe { + var activeHealth string + switch containerName { + case string(apicommon.CoreAgentContainerName): + activeHealth = "exec /opt/datadog-agent/bin/agent/agent health" + case string(apicommon.TraceAgentContainerName): + activeHealth = "exec 3<>/dev/tcp/127.0.0.1/8126; exec 3>&-; exec 3<&-" + default: + activeHealth = "exit 1" + } + waiting := "" + if allowWaiting { + waiting = "prepared|activating) exit 0;; " + } + command := fmt.Sprintf(`state="$(cat %s 2>/dev/null)" || exit 1; case "$state" in %sactive) %s;; *) exit 1;; esac`, statePath, waiting, activeHealth) + + probe := &corev1.Probe{PeriodSeconds: 10, TimeoutSeconds: 1, FailureThreshold: 3} + if base != nil { + probe = base.DeepCopy() + } + probe.ProbeHandler = corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"bash", "-c", command}}} + return probe +} + type preparedHandoffCandidate struct { replacement *corev1.Pod old *corev1.Pod diff --git a/internal/controller/datadogagentinternal/prepared_rollout_test.go b/internal/controller/datadogagentinternal/prepared_rollout_test.go index ed5f74f632..b2efde6a70 100644 --- a/internal/controller/datadogagentinternal/prepared_rollout_test.go +++ b/internal/controller/datadogagentinternal/prepared_rollout_test.go @@ -6,6 +6,7 @@ package datadogagentinternal import ( "context" + "strings" "testing" "time" @@ -40,6 +41,10 @@ func TestPrepareAgentTemplate(t *testing.T) { assert.Contains(t, envValue(container.Env, rolloutStatePathEnv), container.Name+".state") } assert.Equal(t, "trace-agent", arm.Spec.Template.Spec.Containers[1].Command[0]) + assert.Contains(t, strings.Join(arm.Spec.Template.Spec.Containers[0].LivenessProbe.Exec.Command, " "), "agent health") + assert.Contains(t, strings.Join(arm.Spec.Template.Spec.Containers[1].LivenessProbe.Exec.Command, " "), "/dev/tcp/127.0.0.1/8126") + assert.NotContains(t, strings.Join(arm.Spec.Template.Spec.Containers[0].StartupProbe.Exec.Command, " "), "agent health", + "the sleeping replacement startup probe must not contact the old host-network listener") standby := preparedRolloutDaemonSet() require.NoError(t, prepareAgentTemplate(standby, preparedRolloutPhaseStandby)) From 3a5ce935ea8c66b3789e5b5bd9ccd392ac2b3ae3 Mon Sep 17 00:00:00 2001 From: "ali.benabdallah" Date: Wed, 22 Jul 2026 18:37:09 +0200 Subject: [PATCH 04/16] Make rollout arming resilient to API defaults --- .../datadogagentinternal/prepared_rollout.go | 38 +++++++++++++++---- .../prepared_rollout_test.go | 3 ++ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/internal/controller/datadogagentinternal/prepared_rollout.go b/internal/controller/datadogagentinternal/prepared_rollout.go index bd72dd9fb2..88e735d58e 100644 --- a/internal/controller/datadogagentinternal/prepared_rollout.go +++ b/internal/controller/datadogagentinternal/prepared_rollout.go @@ -6,6 +6,8 @@ package datadogagentinternal import ( "context" + "crypto/sha256" + "encoding/json" "fmt" "sort" "strings" @@ -13,7 +15,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -25,11 +26,12 @@ import ( ) const ( - preparedRolloutAnnotation = "experimental.agent.datadoghq.com/host-network-surge-prepared" - preparedRolloutPhaseAnnotation = "experimental.agent.datadoghq.com/prepared-rollout-phase" - resourceFallbackAnnotation = "experimental.agent.datadoghq.com/resource-fallback" - preparedRolloutPhaseArm = "arm" - preparedRolloutPhaseStandby = "standby" + preparedRolloutAnnotation = "experimental.agent.datadoghq.com/host-network-surge-prepared" + preparedRolloutPhaseAnnotation = "experimental.agent.datadoghq.com/prepared-rollout-phase" + preparedRolloutArmHashAnnotation = "experimental.agent.datadoghq.com/prepared-rollout-arm-hash" + resourceFallbackAnnotation = "experimental.agent.datadoghq.com/resource-fallback" + preparedRolloutPhaseArm = "arm" + preparedRolloutPhaseStandby = "standby" preparedRolloutLockVolume = "agent-rollout-locks" preparedRolloutStateVolume = "agent-rollout-state" @@ -72,10 +74,14 @@ func (r *Reconciler) configurePreparedRollout(ctx context.Context, ddai *datadog if err := prepareAgentTemplate(armed, preparedRolloutPhaseArm); err != nil { return "", err } + armHash, err := stampPreparedArmHash(&armed.Spec.Template) + if err != nil { + return "", err + } configureArmStrategy(armed, budget) live := &appsv1.DaemonSet{} - err := r.apiReader.Get(ctx, client.ObjectKeyFromObject(desired), live) + err = r.apiReader.Get(ctx, client.ObjectKeyFromObject(desired), live) if err != nil && !apierrors.IsNotFound(err) { return "", fmt.Errorf("get live Agent DaemonSet for prepared rollout: %w", err) } @@ -88,7 +94,7 @@ func (r *Reconciler) configurePreparedRollout(ctx context.Context, ddai *datadog // Never oscillate back to arm during a mixed or failed surged rollout. phase = preparedRolloutPhaseStandby case preparedRolloutPhaseArm: - if apiequality.Semantic.DeepEqual(live.Spec.Template, armed.Spec.Template) && daemonSetArmComplete(live) { + if live.Spec.Template.Annotations[preparedRolloutArmHashAnnotation] == armHash && daemonSetArmComplete(live) { phase = preparedRolloutPhaseStandby } } @@ -103,6 +109,7 @@ func (r *Reconciler) configurePreparedRollout(ctx context.Context, ddai *datadog if err := prepareAgentTemplate(standby, preparedRolloutPhaseStandby); err != nil { return "", err } + standby.Spec.Template.Annotations[preparedRolloutArmHashAnnotation] = armHash if !configureResourceFallback(standby, budget) { return "", fmt.Errorf("prepared Agent rollout requires a positive, valid maxUnavailable budget") } @@ -110,6 +117,21 @@ func (r *Reconciler) configurePreparedRollout(ctx context.Context, ddai *datadog return phase, nil } +func stampPreparedArmHash(template *corev1.PodTemplateSpec) (string, error) { + templateCopy := template.DeepCopy() + delete(templateCopy.Annotations, preparedRolloutArmHashAnnotation) + serialized, err := json.Marshal(templateCopy) + if err != nil { + return "", fmt.Errorf("hash prepared Agent arm template: %w", err) + } + hash := fmt.Sprintf("%x", sha256.Sum256(serialized)) + if template.Annotations == nil { + template.Annotations = map[string]string{} + } + template.Annotations[preparedRolloutArmHashAnnotation] = hash + return hash, nil +} + func daemonSetArmComplete(ds *appsv1.DaemonSet) bool { desired := ds.Status.DesiredNumberScheduled return desired > 0 && diff --git a/internal/controller/datadogagentinternal/prepared_rollout_test.go b/internal/controller/datadogagentinternal/prepared_rollout_test.go index b2efde6a70..0e56ddbca0 100644 --- a/internal/controller/datadogagentinternal/prepared_rollout_test.go +++ b/internal/controller/datadogagentinternal/prepared_rollout_test.go @@ -68,6 +68,9 @@ func TestConfigurePreparedRolloutArmsThenStaysInStandby(t *testing.T) { assert.Equal(t, intstr.FromInt(0), *desired.Spec.UpdateStrategy.RollingUpdate.MaxSurge) live := desired.DeepCopy() + // The API server defaults Pod fields on the live DaemonSet. Phase progress + // must use the controller-owned desired-template hash, not raw equality. + live.Spec.Template.Spec.DNSPolicy = corev1.DNSClusterFirst live.UID = types.UID("daemonset-uid") live.Generation = 2 live.Status = appsv1.DaemonSetStatus{ From cad10b1534e645f8843308ccdd35b131d60f0f47 Mon Sep 17 00:00:00 2001 From: "ali.benabdallah" Date: Thu, 23 Jul 2026 11:46:01 +0200 Subject: [PATCH 05/16] Poll incomplete prepared rollout arming --- .../controller_reconcile_agent.go | 1 + .../datadogagentinternal/prepared_rollout.go | 11 ++++++++++ .../prepared_rollout_test.go | 21 +++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/internal/controller/datadogagentinternal/controller_reconcile_agent.go b/internal/controller/datadogagentinternal/controller_reconcile_agent.go index 51412d3af8..8552e55402 100644 --- a/internal/controller/datadogagentinternal/controller_reconcile_agent.go +++ b/internal/controller/datadogagentinternal/controller_reconcile_agent.go @@ -304,6 +304,7 @@ func (r *Reconciler) reconcileV2Agent(ctx context.Context, requiredComponents fe if err != nil || preparedPhase == "" { return result, err } + requeuePreparedArm(&result, preparedPhase) if preparedPhase == preparedRolloutPhaseStandby { handoffResult, handoffErr := r.reconcilePreparedHandoff(ctx, ddai, daemonset, rolloutBudget) if handoffErr != nil { diff --git a/internal/controller/datadogagentinternal/prepared_rollout.go b/internal/controller/datadogagentinternal/prepared_rollout.go index 88e735d58e..1f619a1ca1 100644 --- a/internal/controller/datadogagentinternal/prepared_rollout.go +++ b/internal/controller/datadogagentinternal/prepared_rollout.go @@ -37,6 +37,7 @@ const ( preparedRolloutStateVolume = "agent-rollout-state" preparedRolloutLockDir = "/var/run/datadog-agent-rollout" preparedRolloutStateDir = "/var/run/datadog-agent-rollout-state" + preparedRolloutRequeue = time.Second rolloutEnabledEnv = "DD_EXPERIMENTAL_NODE_AGENT_ROLLOUT_ENABLED" rolloutLockPathEnv = "DD_EXPERIMENTAL_NODE_AGENT_ROLLOUT_LOCK_PATH" @@ -117,6 +118,16 @@ func (r *Reconciler) configurePreparedRollout(ctx context.Context, ddai *datadog return phase, nil } +// requeuePreparedArm keeps polling until the DaemonSet controller reports the +// fully available arm revision. DaemonSet status-only updates are filtered by +// the controller watch, so Pod and generation events alone cannot guarantee a +// final reconcile after status catches up. +func requeuePreparedArm(result *reconcile.Result, phase string) { + if phase == preparedRolloutPhaseArm && (result.RequeueAfter == 0 || preparedRolloutRequeue < result.RequeueAfter) { + result.RequeueAfter = preparedRolloutRequeue + } +} + func stampPreparedArmHash(template *corev1.PodTemplateSpec) (string, error) { templateCopy := template.DeepCopy() delete(templateCopy.Annotations, preparedRolloutArmHashAnnotation) diff --git a/internal/controller/datadogagentinternal/prepared_rollout_test.go b/internal/controller/datadogagentinternal/prepared_rollout_test.go index 0e56ddbca0..58ef60c30d 100644 --- a/internal/controller/datadogagentinternal/prepared_rollout_test.go +++ b/internal/controller/datadogagentinternal/prepared_rollout_test.go @@ -20,6 +20,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" "github.com/stretchr/testify/assert" @@ -114,6 +115,26 @@ func TestConfigurePreparedRolloutRejectsUnsupportedContainerWithoutMutation(t *t assert.Equal(t, original, desired) } +func TestRequeuePreparedArm(t *testing.T) { + t.Run("polls while arm status is incomplete", func(t *testing.T) { + result := reconcile.Result{} + requeuePreparedArm(&result, preparedRolloutPhaseArm) + assert.Equal(t, time.Second, result.RequeueAfter) + }) + + t.Run("keeps an earlier requeue", func(t *testing.T) { + result := reconcile.Result{RequeueAfter: 500 * time.Millisecond} + requeuePreparedArm(&result, preparedRolloutPhaseArm) + assert.Equal(t, 500*time.Millisecond, result.RequeueAfter) + }) + + t.Run("does not poll once standby starts", func(t *testing.T) { + result := reconcile.Result{} + requeuePreparedArm(&result, preparedRolloutPhaseStandby) + assert.Zero(t, result.RequeueAfter) + }) +} + func TestPodPreparedForHandoff(t *testing.T) { pod := preparedReplacementPod() assert.True(t, podPreparedForHandoff(pod)) From ba2e631422657712c8a4211324c42cf006e0e8f6 Mon Sep 17 00:00:00 2001 From: "ali.benabdallah" Date: Thu, 23 Jul 2026 12:14:39 +0200 Subject: [PATCH 06/16] Refresh third-party licenses --- LICENSE-3rdparty.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 8274a59a43..0c04bd83e3 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -265,7 +265,7 @@ core,k8s.io/client-go,Apache-2.0 core,k8s.io/client-go/third_party/forked/golang/template,BSD-3-Clause core,k8s.io/cloud-provider/api,Apache-2.0 core,k8s.io/component-base,Apache-2.0 -core,k8s.io/component-helpers/resource,Apache-2.0 +core,k8s.io/component-helpers,Apache-2.0 core,k8s.io/csi-translation-lib,Apache-2.0 core,k8s.io/klog/v2,Apache-2.0 core,k8s.io/kube-aggregator/pkg/apis/apiregistration,Apache-2.0 From f1648b6282c372c530a34d06580da4b764b7a56d Mon Sep 17 00:00:00 2001 From: "ali.benabdallah" Date: Thu, 23 Jul 2026 13:56:37 +0200 Subject: [PATCH 07/16] Cover prepared rollout safety orchestration --- .../controller_reconcile_agent_test.go | 130 ++++++++++++++++++ .../prepared_rollout_test.go | 110 +++++++++++++++ .../resource_fallback_test.go | 19 +++ 3 files changed, 259 insertions(+) diff --git a/internal/controller/datadogagentinternal/controller_reconcile_agent_test.go b/internal/controller/datadogagentinternal/controller_reconcile_agent_test.go index d742818953..2a7c815553 100644 --- a/internal/controller/datadogagentinternal/controller_reconcile_agent_test.go +++ b/internal/controller/datadogagentinternal/controller_reconcile_agent_test.go @@ -1,23 +1,153 @@ package datadogagentinternal import ( + "context" "testing" + "time" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/tools/record" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + apicommon "github.com/DataDog/datadog-operator/api/datadoghq/common" datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" datadoghqv2alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v2alpha1" "github.com/DataDog/datadog-operator/internal/controller/datadogagent/component" + "github.com/DataDog/datadog-operator/internal/controller/datadogagent/defaults" + "github.com/DataDog/datadog-operator/internal/controller/datadogagent/feature" + "github.com/DataDog/datadog-operator/internal/controller/datadogagent/store" "github.com/DataDog/datadog-operator/pkg/constants" "github.com/DataDog/datadog-operator/pkg/kubernetes" + pkgtestutils "github.com/DataDog/datadog-operator/pkg/testutils" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const defaultProvider = kubernetes.DefaultProvider const gkeCosProvider = kubernetes.GKECloudProvider + "-" + kubernetes.GKECosType +func TestReconcileV2AgentRejectsInvalidPreparedRolloutBeforeCreatingDaemonSet(t *testing.T) { + r, ddai := newPreparedRolloutReconciler(t, false) + + result, err := r.reconcileV2Agent( + context.Background(), + preparedRolloutRequiredComponents(), + nil, + ddai, + feature.NewResourceManagers(store.NewStore(ddai, nil)), + &datadoghqv1alpha1.DatadogAgentInternalStatus{}, + defaultProvider, + ) + + require.ErrorContains(t, err, "hostNetwork=true") + assert.Zero(t, result.RequeueAfter) + daemonSets := &appsv1.DaemonSetList{} + require.NoError(t, r.client.List(context.Background(), daemonSets)) + assert.Empty(t, daemonSets.Items, "an incompatible prepared rollout must fail before creating a DaemonSet") +} + +func TestReconcileV2AgentCreatesArmedPreparedDaemonSet(t *testing.T) { + r, ddai := newPreparedRolloutReconciler(t, true) + ddai.Annotations[resourceFallbackAnnotation] = "true" + status := &datadoghqv1alpha1.DatadogAgentInternalStatus{} + + result, err := r.reconcileV2Agent( + context.Background(), + preparedRolloutRequiredComponents(), + nil, + ddai, + feature.NewResourceManagers(store.NewStore(ddai, nil)), + status, + defaultProvider, + ) + + require.NoError(t, err) + assert.Equal(t, time.Second, result.RequeueAfter) + daemonSets := &appsv1.DaemonSetList{} + require.NoError(t, r.client.List(context.Background(), daemonSets)) + require.Len(t, daemonSets.Items, 1) + ds := &daemonSets.Items[0] + assert.Equal(t, preparedRolloutPhaseArm, ds.Spec.Template.Annotations[preparedRolloutPhaseAnnotation]) + require.NotNil(t, ds.Spec.UpdateStrategy.RollingUpdate) + assert.Equal(t, intstr.FromInt(0), *ds.Spec.UpdateStrategy.RollingUpdate.MaxSurge) + assert.Equal(t, intstr.FromInt(1), *ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable) + + ds.Status = appsv1.DaemonSetStatus{ + ObservedGeneration: ds.Generation, + DesiredNumberScheduled: 1, + UpdatedNumberScheduled: 1, + NumberReady: 1, + NumberAvailable: 1, + } + require.NoError(t, r.client.Status().Update(context.Background(), ds)) + + result, err = r.reconcileV2Agent( + context.Background(), + preparedRolloutRequiredComponents(), + nil, + ddai, + feature.NewResourceManagers(store.NewStore(ddai, nil)), + status, + defaultProvider, + ) + require.NoError(t, err) + assert.Zero(t, result.RequeueAfter, "standby without a handoff candidate does not need an extra poll") + require.NoError(t, r.client.Get(context.Background(), client.ObjectKeyFromObject(ds), ds)) + assert.Equal(t, preparedRolloutPhaseStandby, ds.Spec.Template.Annotations[preparedRolloutPhaseAnnotation]) + assert.Equal(t, intstr.FromInt(1), *ds.Spec.UpdateStrategy.RollingUpdate.MaxSurge) + assert.Equal(t, intstr.FromInt(0), *ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable) +} + +func newPreparedRolloutReconciler(t *testing.T, hostNetwork bool) (*Reconciler, *datadoghqv1alpha1.DatadogAgentInternal) { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, appsv1.AddToScheme(scheme)) + require.NoError(t, datadoghqv1alpha1.AddToScheme(scheme)) + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&appsv1.DaemonSet{}).Build() + one := intstr.FromInt(1) + ddai := pkgtestutils.NewDatadogAgentInternal("datadog-agent", "agent", nil) + ddai.UID = "ddai-uid" + ddai.Annotations = map[string]string{preparedRolloutAnnotation: "true"} + ddai.Spec.Features = &datadoghqv2alpha1.DatadogFeatures{} + ddai.Spec.Override = map[datadoghqv2alpha1.ComponentName]*datadoghqv2alpha1.DatadogAgentComponentOverride{ + datadoghqv2alpha1.NodeAgentComponentName: { + HostNetwork: ptr.To(hostNetwork), + UpdateStrategy: &apicommon.UpdateStrategy{ + Type: string(appsv1.RollingUpdateDaemonSetStrategyType), + RollingUpdate: &apicommon.RollingUpdate{ + MaxSurge: ptr.To(one), + MaxUnavailable: ptr.To(one), + }, + }, + }, + } + defaults.DefaultDatadogAgentSpec(&ddai.Spec) + return &Reconciler{ + client: c, + apiReader: c, + scheme: scheme, + recorder: record.NewFakeRecorder(10), + }, ddai +} + +func preparedRolloutRequiredComponents() feature.RequiredComponents { + return feature.RequiredComponents{Agent: feature.RequiredComponent{ + IsRequired: ptr.To(true), + Containers: []apicommon.AgentContainerName{ + apicommon.CoreAgentContainerName, + apicommon.TraceAgentContainerName, + }, + }} +} + // func Test_getValidDaemonSetNames(t *testing.T) { // testCases := []struct { // name string diff --git a/internal/controller/datadogagentinternal/prepared_rollout_test.go b/internal/controller/datadogagentinternal/prepared_rollout_test.go index 58ef60c30d..b3ececd386 100644 --- a/internal/controller/datadogagentinternal/prepared_rollout_test.go +++ b/internal/controller/datadogagentinternal/prepared_rollout_test.go @@ -115,6 +115,116 @@ func TestConfigurePreparedRolloutRejectsUnsupportedContainerWithoutMutation(t *t assert.Equal(t, original, desired) } +func TestPrepareAgentTemplateRejectsUnsafeTemplates(t *testing.T) { + tests := []struct { + name string + mutate func(*appsv1.DaemonSet) + wantErr string + }{ + { + name: "windows node selector", + mutate: func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.NodeSelector = map[string]string{corev1.LabelOSStable: "windows"} + }, + wantErr: "Linux-only", + }, + { + name: "container lifecycle hook", + mutate: func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.Containers[0].Lifecycle = &corev1.Lifecycle{} + }, + wantErr: "lifecycle hooks", + }, + { + name: "container arguments", + mutate: func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.Containers[0].Args = []string{"--extra"} + }, + wantErr: "command arguments", + }, + { + name: "nonstandard core command", + mutate: func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.Containers[0].Command = []string{"agent", "start"} + }, + wantErr: "standard agent run command", + }, + { + name: "reserved mount path", + mutate: func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: "custom", MountPath: preparedRolloutLockDir}} + }, + wantErr: "reserved name or path", + }, + { + name: "unknown trace loader", + mutate: func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.Containers[1].Command = []string{"custom-loader"} + }, + wantErr: "unknown trace-agent loader", + }, + { + name: "unexpected init container", + mutate: func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.InitContainers[1].Name = "custom-init" + }, + wantErr: "does not support init container", + }, + { + name: "init container port", + mutate: func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.InitContainers[0].Ports = []corev1.ContainerPort{{ContainerPort: 1234}} + }, + wantErr: "ports or lifecycle hooks", + }, + { + name: "reserved volume name", + mutate: func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.Volumes = []corev1.Volume{{Name: preparedRolloutStateVolume}} + }, + wantErr: "reserved", + }, + { + name: "custom anti-affinity", + mutate: func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.Affinity = &corev1.Affinity{PodAntiAffinity: &corev1.PodAntiAffinity{}} + }, + wantErr: "custom Pod anti-affinity", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ds := preparedRolloutDaemonSet() + test.mutate(ds) + err := prepareAgentTemplate(ds, preparedRolloutPhaseArm) + require.ErrorContains(t, err, test.wantErr) + }) + } +} + +func TestConfigurePreparedContainerReplacesRolloutEnvironment(t *testing.T) { + container := &corev1.Container{ + Name: "agent", + Env: []corev1.EnvVar{ + {Name: rolloutEnabledEnv, Value: "false"}, + {Name: "KEEP_ME", Value: "value"}, + }, + } + + configurePreparedContainer(container) + + assert.Equal(t, "true", envValue(container.Env, rolloutEnabledEnv)) + assert.Equal(t, "value", envValue(container.Env, "KEEP_ME")) + count := 0 + for i := range container.Env { + if container.Env[i].Name == rolloutEnabledEnv { + count++ + } + } + assert.Equal(t, 1, count, "rollout configuration must replace, not duplicate, an existing environment variable") +} + func TestRequeuePreparedArm(t *testing.T) { t.Run("polls while arm status is incomplete", func(t *testing.T) { result := reconcile.Result{} diff --git a/internal/controller/datadogagentinternal/resource_fallback_test.go b/internal/controller/datadogagentinternal/resource_fallback_test.go index 2727d8364b..a3aa376653 100644 --- a/internal/controller/datadogagentinternal/resource_fallback_test.go +++ b/internal/controller/datadogagentinternal/resource_fallback_test.go @@ -26,6 +26,8 @@ import ( datadoghqcommon "github.com/DataDog/datadog-operator/api/datadoghq/common" datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" + datadoghqv2alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v2alpha1" + componentagent "github.com/DataDog/datadog-operator/internal/controller/datadogagent/component/agent" "github.com/DataDog/datadog-operator/pkg/constants" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -91,6 +93,23 @@ func TestConfigureResourceFallback(t *testing.T) { } } +func TestResourceFallbackBudgetPrecedence(t *testing.T) { + overrideBudget := intstr.FromString("25%") + ddai := &datadoghqv1alpha1.DatadogAgentInternal{Spec: datadoghqv2alpha1.DatadogAgentSpec{ + Override: map[datadoghqv2alpha1.ComponentName]*datadoghqv2alpha1.DatadogAgentComponentOverride{ + datadoghqv2alpha1.NodeAgentComponentName: { + UpdateStrategy: &datadoghqcommon.UpdateStrategy{RollingUpdate: &datadoghqcommon.RollingUpdate{MaxUnavailable: &overrideBudget}}, + }, + }, + }} + options := &componentagent.ExtendedDaemonsetOptions{MaxPodUnavailable: "2"} + + assert.Equal(t, overrideBudget, resourceFallbackBudget(ddai, options), "the DatadogAgent override is the requested rollout budget") + ddai.Spec.Override = nil + assert.Equal(t, intstr.FromInt(2), resourceFallbackBudget(ddai, options), "the Operator option is the compatibility fallback") + assert.Equal(t, intstr.FromInt(defaultFallbackMaxUnavailable), resourceFallbackBudget(ddai, nil), "the fallback remains bounded when neither source is configured") +} + func TestResourceOnlyUnschedulable(t *testing.T) { tests := []struct { name string From 1079119c28e7ff5a8ad4647b09979b66cc93d244 Mon Sep 17 00:00:00 2001 From: "ali.benabdallah" Date: Thu, 23 Jul 2026 14:32:37 +0200 Subject: [PATCH 08/16] Fix experimental rollout event handling --- .../controller_reconcile_agent.go | 3 +- .../prepared_rollout_test.go | 128 ++++++++++++++++++ .../datadogagentinternal_controller.go | 33 +++-- .../datadogagentinternal_controller_test.go | 112 +++++++++++++++ 4 files changed, 261 insertions(+), 15 deletions(-) diff --git a/internal/controller/datadogagentinternal/controller_reconcile_agent.go b/internal/controller/datadogagentinternal/controller_reconcile_agent.go index 8552e55402..36147f414d 100644 --- a/internal/controller/datadogagentinternal/controller_reconcile_agent.go +++ b/internal/controller/datadogagentinternal/controller_reconcile_agent.go @@ -38,7 +38,6 @@ import ( func (r *Reconciler) reconcileV2Agent(ctx context.Context, requiredComponents feature.RequiredComponents, features []feature.Feature, ddai *datadoghqv1alpha1.DatadogAgentInternal, resourcesManager feature.ResourceManagers, newStatus *datadoghqv1alpha1.DatadogAgentInternalStatus, provider string) (reconcile.Result, error) { var result reconcile.Result - var err error var eds *edsv1alpha1.ExtendedDaemonSet var daemonset *appsv1.DaemonSet var podManagers feature.PodTemplateManagers @@ -300,7 +299,7 @@ func (r *Reconciler) reconcileV2Agent(ctx context.Context, requiredComponents fe } return reconcile.Result{}, prepareErr } - result, err = r.createOrUpdateDaemonset(ctx, ddai, daemonset, newStatus, updateDSStatusV2WithAgent) + result, err := r.createOrUpdateDaemonset(ctx, ddai, daemonset, newStatus, updateDSStatusV2WithAgent) if err != nil || preparedPhase == "" { return result, err } diff --git a/internal/controller/datadogagentinternal/prepared_rollout_test.go b/internal/controller/datadogagentinternal/prepared_rollout_test.go index b3ececd386..136b29eb16 100644 --- a/internal/controller/datadogagentinternal/prepared_rollout_test.go +++ b/internal/controller/datadogagentinternal/prepared_rollout_test.go @@ -22,6 +22,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" + apicommon "github.com/DataDog/datadog-operator/api/datadoghq/common" datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -115,12 +116,60 @@ func TestConfigurePreparedRolloutRejectsUnsupportedContainerWithoutMutation(t *t assert.Equal(t, original, desired) } +func TestConfigurePreparedRolloutRejectsInvalidGates(t *testing.T) { + scheme := runtime.NewScheme() + reconciler := &Reconciler{apiReader: fake.NewClientBuilder().WithScheme(scheme).Build()} + enabled := &datadoghqv1alpha1.DatadogAgentInternal{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{preparedRolloutAnnotation: "true"}}} + + disabled := &datadoghqv1alpha1.DatadogAgentInternal{} + desired := preparedRolloutDaemonSet() + original := desired.DeepCopy() + phase, err := reconciler.configurePreparedRollout(context.Background(), disabled, desired, intstr.FromInt(1)) + require.NoError(t, err) + assert.Empty(t, phase) + assert.Equal(t, original, desired) + + _, err = reconciler.configurePreparedRollout(context.Background(), enabled, preparedRolloutDaemonSet(), intstr.FromInt(0)) + require.ErrorContains(t, err, "positive, valid maxUnavailable") + + onDelete := preparedRolloutDaemonSet() + onDelete.Spec.UpdateStrategy.Type = appsv1.OnDeleteDaemonSetStrategyType + _, err = reconciler.configurePreparedRollout(context.Background(), enabled, onDelete, intstr.FromInt(1)) + require.ErrorContains(t, err, "RollingUpdate strategy") +} + +func TestConfigureArmStrategyInitializesRollingUpdate(t *testing.T) { + ds := &appsv1.DaemonSet{} + configureArmStrategy(ds, intstr.FromString("25%")) + require.NotNil(t, ds.Spec.UpdateStrategy.RollingUpdate) + assert.Equal(t, intstr.FromInt(0), *ds.Spec.UpdateStrategy.RollingUpdate.MaxSurge) + assert.Equal(t, intstr.FromString("25%"), *ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable) + + previous := ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable + configureArmStrategy(ds, intstr.FromInt(0)) + assert.Equal(t, previous, ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable) +} + func TestPrepareAgentTemplateRejectsUnsafeTemplates(t *testing.T) { tests := []struct { name string mutate func(*appsv1.DaemonSet) wantErr string }{ + { + name: "host network disabled", + mutate: func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.HostNetwork = false + }, + wantErr: "hostNetwork=true", + }, + { + name: "windows pod os", + mutate: func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.OS = &corev1.PodOS{Name: corev1.Windows} + }, + wantErr: "Linux-only", + }, { name: "windows node selector", mutate: func(ds *appsv1.DaemonSet) { @@ -128,6 +177,34 @@ func TestPrepareAgentTemplateRejectsUnsafeTemplates(t *testing.T) { }, wantErr: "Linux-only", }, + { + name: "legacy windows node selector", + mutate: func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.NodeSelector = map[string]string{"beta.kubernetes.io/os": "windows"} + }, + wantErr: "Linux-only", + }, + { + name: "missing trace container", + mutate: func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.Containers = ds.Spec.Template.Spec.Containers[:1] + }, + wantErr: "exactly agent and trace-agent", + }, + { + name: "unsupported container", + mutate: func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.Containers[1].Name = "security-agent" + }, + wantErr: "does not support container", + }, + { + name: "duplicate container", + mutate: func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.Containers[1].Name = string(apicommon.CoreAgentContainerName) + }, + wantErr: "duplicate container", + }, { name: "container lifecycle hook", mutate: func(ds *appsv1.DaemonSet) { @@ -156,6 +233,13 @@ func TestPrepareAgentTemplateRejectsUnsafeTemplates(t *testing.T) { }, wantErr: "reserved name or path", }, + { + name: "reserved mount name", + mutate: func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: preparedRolloutLockVolume, MountPath: "/custom"}} + }, + wantErr: "reserved name or path", + }, { name: "unknown trace loader", mutate: func(ds *appsv1.DaemonSet) { @@ -170,6 +254,27 @@ func TestPrepareAgentTemplateRejectsUnsafeTemplates(t *testing.T) { }, wantErr: "does not support init container", }, + { + name: "missing init container", + mutate: func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.InitContainers = ds.Spec.Template.Spec.InitContainers[:1] + }, + wantErr: "only init-volume and init-config", + }, + { + name: "duplicate init container", + mutate: func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.InitContainers[1].Name = string(apicommon.InitVolumeContainerName) + }, + wantErr: "requires init-volume and init-config", + }, + { + name: "init container lifecycle hook", + mutate: func(ds *appsv1.DaemonSet) { + ds.Spec.Template.Spec.InitContainers[0].Lifecycle = &corev1.Lifecycle{} + }, + wantErr: "ports or lifecycle hooks", + }, { name: "init container port", mutate: func(ds *appsv1.DaemonSet) { @@ -253,6 +358,29 @@ func TestPodPreparedForHandoff(t *testing.T) { pod.Status.ContainerStatuses[1].Started = ptr.To(true) pod.Status.ContainerStatuses[1].RestartCount = 1 assert.False(t, podPreparedForHandoff(pod)) + + tests := []struct { + name string + mutate func(*corev1.Pod) + }{ + {name: "deleting", mutate: func(p *corev1.Pod) { now := metav1.Now(); p.DeletionTimestamp = &now }}, + {name: "not running", mutate: func(p *corev1.Pod) { p.Status.Phase = corev1.PodPending }}, + {name: "missing init status", mutate: func(p *corev1.Pod) { p.Status.InitContainerStatuses = p.Status.InitContainerStatuses[:1] }}, + {name: "missing container status", mutate: func(p *corev1.Pod) { p.Status.ContainerStatuses = p.Status.ContainerStatuses[:1] }}, + {name: "failed init", mutate: func(p *corev1.Pod) { p.Status.InitContainerStatuses[0].State.Terminated.ExitCode = 1 }}, + {name: "running init", mutate: func(p *corev1.Pod) { p.Status.InitContainerStatuses[0].State.Terminated = nil }}, + {name: "unknown container", mutate: func(p *corev1.Pod) { p.Status.ContainerStatuses[0].Name = "security-agent" }}, + {name: "container stopped", mutate: func(p *corev1.Pod) { p.Status.ContainerStatuses[0].State.Running = nil }}, + {name: "started unknown", mutate: func(p *corev1.Pod) { p.Status.ContainerStatuses[0].Started = nil }}, + {name: "duplicate agent", mutate: func(p *corev1.Pod) { p.Status.ContainerStatuses[1].Name = "agent" }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + candidate := preparedReplacementPod() + test.mutate(candidate) + assert.False(t, podPreparedForHandoff(candidate)) + }) + } } func TestReconcilePreparedHandoffReservesThenDeletesOldPod(t *testing.T) { diff --git a/internal/controller/datadogagentinternal_controller.go b/internal/controller/datadogagentinternal_controller.go index ace8799025..3f7e2bf855 100644 --- a/internal/controller/datadogagentinternal_controller.go +++ b/internal/controller/datadogagentinternal_controller.go @@ -66,23 +66,23 @@ func (r *DatadogAgentInternalReconciler) Reconcile(ctx context.Context, ddai *v1 func (r *DatadogAgentInternalReconciler) SetupWithManager(mgr ctrl.Manager, metricForwardersMgr datadog.MetricsForwardersManager) error { generationChanged := ctrlbuilder.WithPredicates(predicate.GenerationChangedPredicate{}) builder := ctrl.NewControllerManagedBy(mgr). - Owns(&corev1.Secret{}, generationChanged). - Owns(&corev1.ConfigMap{}, generationChanged). + Owns(&corev1.Secret{}). + Owns(&corev1.ConfigMap{}). Owns(&appsv1.DaemonSet{}, generationChanged). - Owns(&appsv1.Deployment{}, generationChanged). - Owns(&rbacv1.Role{}, generationChanged). - Owns(&rbacv1.RoleBinding{}, generationChanged). - Owns(&corev1.ServiceAccount{}, generationChanged). + Owns(&appsv1.Deployment{}). + Owns(&rbacv1.Role{}). + Owns(&rbacv1.RoleBinding{}). + Owns(&corev1.ServiceAccount{}). // We let PlatformInfo supply PDB object based on the current API version - Owns(r.PlatformInfo.CreatePDBObject(), generationChanged). - Owns(&networkingv1.NetworkPolicy{}, generationChanged) + Owns(r.PlatformInfo.CreatePDBObject()). + Owns(&networkingv1.NetworkPolicy{}) // DatadogAgent is namespaced whereas ClusterRole and ClusterRoleBinding are // cluster-scoped. That means that DatadogAgent cannot be their owner, and // we cannot use .Owns(). handlerEnqueue := handler.EnqueueRequestsFromMapFunc(enqueueIfOwnedByDatadogAgentInternal) - builder.Watches(&rbacv1.ClusterRole{}, handlerEnqueue, generationChanged) - builder.Watches(&rbacv1.ClusterRoleBinding{}, handlerEnqueue, generationChanged) + builder.Watches(&rbacv1.ClusterRole{}, handlerEnqueue) + builder.Watches(&rbacv1.ClusterRoleBinding{}, handlerEnqueue) builder.Watches( &corev1.Pod{}, handler.EnqueueRequestsFromMapFunc(enqueueDatadogAgentInternalForPod(mgr.GetAPIReader())), @@ -90,7 +90,7 @@ func (r *DatadogAgentInternalReconciler) SetupWithManager(mgr ctrl.Manager, metr ) if r.Options.ExtendedDaemonsetOptions.Enabled { - builder = builder.Owns(&edsdatadoghqv1alpha1.ExtendedDaemonSet{}, generationChanged) + builder = builder.Owns(&edsdatadoghqv1alpha1.ExtendedDaemonSet{}) } if r.Options.SupportCilium { @@ -100,7 +100,7 @@ func (r *DatadogAgentInternalReconciler) SetupWithManager(mgr ctrl.Manager, metr Version: "v2", Kind: "CiliumNetworkPolicy", }) - builder = builder.Owns(policy, generationChanged) + builder = builder.Owns(policy) } var builderOptions []ctrlbuilder.ForOption @@ -113,7 +113,7 @@ func (r *DatadogAgentInternalReconciler) SetupWithManager(mgr ctrl.Manager, metr }, })) } - builderOptions = append(builderOptions, ctrlbuilder.WithPredicates(predicate.GenerationChangedPredicate{})) + builderOptions = append(builderOptions, ctrlbuilder.WithPredicates(datadogAgentInternalEventPredicate())) or := reconcile.AsReconciler[*v1alpha1.DatadogAgentInternal](r.Client, r) if err := builder.For(&datadoghqv1alpha1.DatadogAgentInternal{}, builderOptions...).Complete(or); err != nil { @@ -127,6 +127,13 @@ func (r *DatadogAgentInternalReconciler) SetupWithManager(mgr ctrl.Manager, metr return nil } +func datadogAgentInternalEventPredicate() predicate.Predicate { + return predicate.Or( + predicate.GenerationChangedPredicate{}, + datadogAnnotationChangedPredicate(), + ) +} + func enqueueDatadogAgentInternalForPod(reader client.Reader) handler.MapFunc { return func(ctx context.Context, obj client.Object) []reconcile.Request { pod, ok := obj.(*corev1.Pod) diff --git a/internal/controller/datadogagentinternal_controller_test.go b/internal/controller/datadogagentinternal_controller_test.go index 7ddd48ab2b..ffb09c4356 100644 --- a/internal/controller/datadogagentinternal_controller_test.go +++ b/internal/controller/datadogagentinternal_controller_test.go @@ -20,13 +20,17 @@ import ( apicommon "github.com/DataDog/datadog-operator/api/datadoghq/common" datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" "github.com/DataDog/datadog-operator/pkg/constants" + "github.com/DataDog/datadog-operator/pkg/kubernetes" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestResourceFallbackPodPredicate(t *testing.T) { predicate := resourceFallbackPodPredicate() + assert.False(t, predicate.Create(event.CreateEvent{Object: &corev1.ConfigMap{}})) + assert.False(t, predicate.Create(event.CreateEvent{Object: &corev1.Pod{}})) oldPod := &corev1.Pod{Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: corev1.PodReasonUnschedulable, Message: "old"}}}} + assert.True(t, predicate.Create(event.CreateEvent{Object: oldPod})) newPod := oldPod.DeepCopy() newPod.Status.Conditions[0].Message = "0/1 nodes are available: 1 Insufficient cpu." assert.True(t, predicate.Update(event.UpdateEvent{ObjectOld: oldPod, ObjectNew: newPod}), "PodScheduled message-only updates must enqueue") @@ -40,6 +44,76 @@ func TestResourceFallbackPodPredicate(t *testing.T) { prepared := waiting.DeepCopy() prepared.Status.ContainerStatuses[0].Started = ptr.To(true) assert.True(t, predicate.Update(event.UpdateEvent{ObjectOld: waiting, ObjectNew: prepared}), "startup-probe Prepared transitions must enqueue the handoff controller") + assert.False(t, predicate.Update(event.UpdateEvent{ObjectOld: &corev1.ConfigMap{}, ObjectNew: &corev1.ConfigMap{}})) + assert.True(t, predicate.Delete(event.DeleteEvent{Object: oldPod})) + assert.False(t, predicate.Generic(event.GenericEvent{Object: oldPod})) +} + +func TestContainerRolloutStatusChanged(t *testing.T) { + running := &corev1.ContainerStateRunning{} + terminated := &corev1.ContainerStateTerminated{} + tests := []struct { + name string + old []corev1.ContainerStatus + new []corev1.ContainerStatus + want bool + }{ + {name: "identical empty", want: false}, + {name: "length", new: []corev1.ContainerStatus{{Name: "agent"}}, want: true}, + {name: "name", old: []corev1.ContainerStatus{{Name: "agent"}}, new: []corev1.ContainerStatus{{Name: "trace-agent"}}, want: true}, + {name: "started", old: []corev1.ContainerStatus{{Name: "agent"}}, new: []corev1.ContainerStatus{{Name: "agent", Started: ptr.To(false)}}, want: true}, + {name: "restart", old: []corev1.ContainerStatus{{Name: "agent"}}, new: []corev1.ContainerStatus{{Name: "agent", RestartCount: 1}}, want: true}, + {name: "running", old: []corev1.ContainerStatus{{Name: "agent"}}, new: []corev1.ContainerStatus{{Name: "agent", State: corev1.ContainerState{Running: running}}}, want: true}, + {name: "terminated", old: []corev1.ContainerStatus{{Name: "agent"}}, new: []corev1.ContainerStatus{{Name: "agent", State: corev1.ContainerState{Terminated: terminated}}}, want: true}, + {name: "identical", old: []corev1.ContainerStatus{{Name: "agent", Started: ptr.To(true), RestartCount: 1, State: corev1.ContainerState{Running: running}}}, new: []corev1.ContainerStatus{{Name: "agent", Started: ptr.To(true), RestartCount: 1, State: corev1.ContainerState{Running: running}}}, want: false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.want, containerRolloutStatusChanged(test.old, test.new)) + }) + } +} + +func TestResourceFallbackConditionChanged(t *testing.T) { + empty := &corev1.Pod{} + scheduled := &corev1.Pod{Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: "Unschedulable", Message: "cpu"}}}} + assert.False(t, resourceFallbackConditionChanged(empty, empty, corev1.PodScheduled)) + assert.True(t, resourceFallbackConditionChanged(empty, scheduled, corev1.PodScheduled)) + assert.False(t, resourceFallbackConditionChanged(scheduled, scheduled.DeepCopy(), corev1.PodScheduled)) + for _, mutate := range []func(*corev1.PodCondition){ + func(condition *corev1.PodCondition) { condition.Status = corev1.ConditionTrue }, + func(condition *corev1.PodCondition) { condition.Reason = "Scheduled" }, + func(condition *corev1.PodCondition) { condition.Message = "memory" }, + } { + changed := scheduled.DeepCopy() + mutate(&changed.Status.Conditions[0]) + assert.True(t, resourceFallbackConditionChanged(scheduled, changed, corev1.PodScheduled)) + } +} + +func TestDatadogAgentInternalEventPredicate(t *testing.T) { + p := datadogAgentInternalEventPredicate() + old := &datadoghqv1alpha1.DatadogAgentInternal{ObjectMeta: metav1.ObjectMeta{Generation: 1, Annotations: map[string]string{"example.com/ignored": "old"}}} + + generation := old.DeepCopy() + generation.Generation = 2 + assert.True(t, p.Update(event.UpdateEvent{ObjectOld: old, ObjectNew: generation})) + + prepared := old.DeepCopy() + prepared.Annotations["experimental.agent.datadoghq.com/host-network-surge-prepared"] = "true" + assert.True(t, p.Update(event.UpdateEvent{ObjectOld: old, ObjectNew: prepared})) + + fallback := prepared.DeepCopy() + fallback.Annotations["experimental.agent.datadoghq.com/resource-fallback"] = "true" + assert.True(t, p.Update(event.UpdateEvent{ObjectOld: prepared, ObjectNew: fallback})) + + removed := fallback.DeepCopy() + delete(removed.Annotations, "experimental.agent.datadoghq.com/resource-fallback") + assert.True(t, p.Update(event.UpdateEvent{ObjectOld: fallback, ObjectNew: removed})) + + unrelated := old.DeepCopy() + unrelated.Annotations["example.com/ignored"] = "new" + assert.False(t, p.Update(event.UpdateEvent{ObjectOld: old, ObjectNew: unrelated})) } func TestEnqueueDatadogAgentInternalForPodFollowsDaemonSetOwner(t *testing.T) { @@ -78,4 +152,42 @@ func TestEnqueueDatadogAgentInternalForPodFollowsDaemonSetOwner(t *testing.T) { pod.OwnerReferences[0].UID = "wrong-uid" assert.Empty(t, enqueueDatadogAgentInternalForPod(reader)(context.Background(), pod), "stale Pod owner UIDs must not enqueue") + + assert.Empty(t, enqueueDatadogAgentInternalForPod(reader)(context.Background(), &corev1.ConfigMap{})) + withoutLabel := pod.DeepCopy() + withoutLabel.Labels = nil + assert.Empty(t, enqueueDatadogAgentInternalForPod(reader)(context.Background(), withoutLabel)) + withoutOwner := pod.DeepCopy() + withoutOwner.OwnerReferences = nil + assert.Empty(t, enqueueDatadogAgentInternalForPod(reader)(context.Background(), withoutOwner)) + wrongOwnerKind := pod.DeepCopy() + wrongOwnerKind.OwnerReferences[0].Kind = "Deployment" + assert.Empty(t, enqueueDatadogAgentInternalForPod(reader)(context.Background(), wrongOwnerKind)) + missingDaemonSet := pod.DeepCopy() + missingDaemonSet.OwnerReferences[0].Name = "missing" + assert.Empty(t, enqueueDatadogAgentInternalForPod(reader)(context.Background(), missingDaemonSet)) + + dsWithoutOwner := ds.DeepCopy() + dsWithoutOwner.Name = "unowned-agent" + dsWithoutOwner.UID = "unowned-ds-uid" + dsWithoutOwner.OwnerReferences = nil + unownedReader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(dsWithoutOwner).Build() + unownedPod := pod.DeepCopy() + unownedPod.OwnerReferences[0].Name = dsWithoutOwner.Name + unownedPod.OwnerReferences[0].UID = dsWithoutOwner.UID + assert.Empty(t, enqueueDatadogAgentInternalForPod(unownedReader)(context.Background(), unownedPod)) +} + +func TestEnqueueIfOwnedByDatadogAgentInternal(t *testing.T) { + unmanaged := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{ + kubernetes.AppKubernetesPartOfLabelKey: "default-profile--ddai", + }}} + assert.Empty(t, enqueueIfOwnedByDatadogAgentInternal(context.Background(), unmanaged)) + + managed := unmanaged.DeepCopy() + managed.Labels[kubernetes.AppKubernetesManageByLabelKey] = "datadog-operator" + requests := enqueueIfOwnedByDatadogAgentInternal(context.Background(), managed) + require.Len(t, requests, 1) + assert.Equal(t, "default", requests[0].Namespace) + assert.Equal(t, "profile-ddai", requests[0].Name) } From 28b0c1e53a13f5e700cb9cecce3e5ebd4b10c4f6 Mon Sep 17 00:00:00 2001 From: "ali.benabdallah" Date: Thu, 23 Jul 2026 14:58:54 +0200 Subject: [PATCH 09/16] Improve experimental rollout safety coverage --- .../resource_fallback_test.go | 127 ++++++++++++++++-- .../datadogagentinternal_controller_test.go | 66 +++++++++ 2 files changed, 181 insertions(+), 12 deletions(-) diff --git a/internal/controller/datadogagentinternal/resource_fallback_test.go b/internal/controller/datadogagentinternal/resource_fallback_test.go index a3aa376653..5e5b64353c 100644 --- a/internal/controller/datadogagentinternal/resource_fallback_test.go +++ b/internal/controller/datadogagentinternal/resource_fallback_test.go @@ -139,18 +139,87 @@ func TestResourceOnlyUnschedulable(t *testing.T) { } func TestTargetNodeFromDaemonSetAffinity(t *testing.T) { - pod := pendingPodForNode("node-a") - got, ok := targetNodeFromDaemonSetAffinity(pod) - require.True(t, ok) - assert.Equal(t, "node-a", got) - - ambiguous := pendingPodForNode("node-a") - ambiguous.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append( - ambiguous.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, - corev1.NodeSelectorTerm{MatchFields: []corev1.NodeSelectorRequirement{{Key: metav1.ObjectNameField, Operator: corev1.NodeSelectorOpIn, Values: []string{"node-b"}}}}, - ) - _, ok = targetNodeFromDaemonSetAffinity(ambiguous) - assert.False(t, ok) + requirement := func(operator corev1.NodeSelectorOperator, values ...string) corev1.NodeSelectorRequirement { + return corev1.NodeSelectorRequirement{Key: metav1.ObjectNameField, Operator: operator, Values: values} + } + podWithTerms := func(terms ...corev1.NodeSelectorTerm) *corev1.Pod { + return &corev1.Pod{Spec: corev1.PodSpec{Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: terms}, + }}}} + } + + tests := []struct { + name string + pod *corev1.Pod + want string + ok bool + }{ + {name: "daemonset target", pod: pendingPodForNode("node-a"), want: "node-a", ok: true}, + {name: "no affinity", pod: &corev1.Pod{}}, + {name: "no node affinity", pod: &corev1.Pod{Spec: corev1.PodSpec{Affinity: &corev1.Affinity{}}}}, + {name: "no required node affinity", pod: &corev1.Pod{Spec: corev1.PodSpec{Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{}}}}}, + {name: "empty terms", pod: podWithTerms()}, + {name: "term without target field", pod: podWithTerms(corev1.NodeSelectorTerm{MatchFields: []corev1.NodeSelectorRequirement{{Key: "metadata.namespace", Operator: corev1.NodeSelectorOpIn, Values: []string{"datadog"}}}})}, + {name: "wrong operator", pod: podWithTerms(corev1.NodeSelectorTerm{MatchFields: []corev1.NodeSelectorRequirement{requirement(corev1.NodeSelectorOpNotIn, "node-a")}})}, + {name: "multiple values", pod: podWithTerms(corev1.NodeSelectorTerm{MatchFields: []corev1.NodeSelectorRequirement{requirement(corev1.NodeSelectorOpIn, "node-a", "node-b")}})}, + {name: "duplicate target field", pod: podWithTerms(corev1.NodeSelectorTerm{MatchFields: []corev1.NodeSelectorRequirement{requirement(corev1.NodeSelectorOpIn, "node-a"), requirement(corev1.NodeSelectorOpIn, "node-a")}})}, + {name: "consistent terms", pod: podWithTerms( + corev1.NodeSelectorTerm{MatchFields: []corev1.NodeSelectorRequirement{requirement(corev1.NodeSelectorOpIn, "node-a")}}, + corev1.NodeSelectorTerm{MatchFields: []corev1.NodeSelectorRequirement{requirement(corev1.NodeSelectorOpIn, "node-a")}}, + ), want: "node-a", ok: true}, + {name: "conflicting terms", pod: podWithTerms( + corev1.NodeSelectorTerm{MatchFields: []corev1.NodeSelectorRequirement{requirement(corev1.NodeSelectorOpIn, "node-a")}}, + corev1.NodeSelectorTerm{MatchFields: []corev1.NodeSelectorRequirement{requirement(corev1.NodeSelectorOpIn, "node-b")}}, + )}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := targetNodeFromDaemonSetAffinity(tt.pod) + assert.Equal(t, tt.ok, ok) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestPrepareProfileAntiAffinityForSurge(t *testing.T) { + labels := map[string]string{ + datadoghqcommon.AgentDeploymentNameLabelKey: "datadog-agent", + constants.ProfileLabelKey: "linux", + } + + t.Run("no anti-affinity", func(t *testing.T) { + for _, template := range []*corev1.PodTemplateSpec{ + {}, + {Spec: corev1.PodSpec{Affinity: &corev1.Affinity{}}}, + } { + assert.True(t, prepareProfileAntiAffinityForSurge(template)) + } + }) + + t.Run("custom anti-affinity is rejected without mutation", func(t *testing.T) { + template := &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Affinity: &corev1.Affinity{PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{TopologyKey: "topology.kubernetes.io/zone"}}, + }}}} + original := template.DeepCopy() + assert.False(t, prepareProfileAntiAffinityForSurge(template)) + assert.Equal(t, original, template) + }) + + t.Run("missing deployment identity is rejected", func(t *testing.T) { + template := &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Affinity: &corev1.Affinity{PodAntiAffinity: broadAgentPodAntiAffinity()}}} + assert.False(t, prepareProfileAntiAffinityForSurge(template)) + }) + + t.Run("standard affinity is narrowed", func(t *testing.T) { + template := &corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{PodAntiAffinity: broadAgentPodAntiAffinity()}, + }} + expected, ok := profileSurgePodAntiAffinity(labels) + require.True(t, ok) + require.True(t, prepareProfileAntiAffinityForSurge(template)) + assert.Equal(t, expected, template.Spec.Affinity.PodAntiAffinity) + }) } func TestResourceFallbackSchedulingShapeAllowsOnlyProfileSurgeAntiAffinity(t *testing.T) { @@ -301,6 +370,40 @@ func TestExistingPodRequiredAntiAffinityCanRejectReplacement(t *testing.T) { assert.False(t, allowed, "wider topology terms fail closed without loading every Node's topology labels") } +func TestPodAffinityTermSelector(t *testing.T) { + term := &corev1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "agent"}}, + MatchLabelKeys: []string{"rollout"}, + MismatchLabelKeys: []string{"profile"}, + } + selector, err := podAffinityTermSelector(term, map[string]string{"rollout": "new", "profile": "linux"}) + require.NoError(t, err) + assert.True(t, selector.Matches(labels.Set{"app": "agent", "rollout": "new", "profile": "gpu"})) + assert.False(t, selector.Matches(labels.Set{"app": "agent", "rollout": "old", "profile": "gpu"})) + assert.False(t, selector.Matches(labels.Set{"app": "agent", "rollout": "new", "profile": "linux"})) + + selector, err = podAffinityTermSelector(term, nil) + require.NoError(t, err) + assert.True(t, selector.Matches(labels.Set{"app": "agent"}), "keys missing from the source Pod must not add selector requirements") + + _, err = podAffinityTermSelector(&corev1.PodAffinityTerm{LabelSelector: &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: "app", Operator: metav1.LabelSelectorOperator("Invalid"), Values: []string{"agent"}, + }}}}, nil) + require.Error(t, err) + + _, err = podAffinityTermSelector(&corev1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{}, + MatchLabelKeys: []string{"bad key"}, + }, map[string]string{"bad key": "value"}) + require.Error(t, err) + + _, err = podAffinityTermSelector(&corev1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{}, + MismatchLabelKeys: []string{"bad key"}, + }, map[string]string{"bad key": "value"}) + require.Error(t, err) +} + func TestResourceFitAfterOldPodRemoval(t *testing.T) { node := &corev1.Node{Status: corev1.NodeStatus{Allocatable: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1500m"), diff --git a/internal/controller/datadogagentinternal_controller_test.go b/internal/controller/datadogagentinternal_controller_test.go index ffb09c4356..27f4aaf494 100644 --- a/internal/controller/datadogagentinternal_controller_test.go +++ b/internal/controller/datadogagentinternal_controller_test.go @@ -8,23 +8,89 @@ import ( "context" "testing" + edsdatadoghqv1alpha1 "github.com/DataDog/extendeddaemonset/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + ctrlconfig "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/manager" apicommon "github.com/DataDog/datadog-operator/api/datadoghq/common" datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" + componentagent "github.com/DataDog/datadog-operator/internal/controller/datadogagent/component/agent" + "github.com/DataDog/datadog-operator/internal/controller/datadogagentinternal" "github.com/DataDog/datadog-operator/pkg/constants" + "github.com/DataDog/datadog-operator/pkg/controller/utils/datadog" "github.com/DataDog/datadog-operator/pkg/kubernetes" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +type noopMetricsForwardersManager struct{} + +func (noopMetricsForwardersManager) Register(client.Object) {} +func (noopMetricsForwardersManager) Unregister(client.Object) {} +func (noopMetricsForwardersManager) ProcessError(client.Object, error) {} +func (noopMetricsForwardersManager) ProcessEvent(client.Object, datadog.Event) {} +func (noopMetricsForwardersManager) SetEnabledFeatures(client.Object, []string) {} +func (noopMetricsForwardersManager) MetricsForwarderStatusForObj(client.Object) *datadog.ConditionCommon { + return nil +} + +func TestDatadogAgentInternalSetupWithManager(t *testing.T) { + tests := []struct { + name string + options datadogagentinternal.ReconcilerOptions + }{ + {name: "default"}, + { + name: "optional watches and metrics", + options: datadogagentinternal.ReconcilerOptions{ + ExtendedDaemonsetOptions: componentagent.ExtendedDaemonsetOptions{Enabled: true}, + SupportCilium: true, + OperatorMetricsEnabled: true, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, datadoghqv1alpha1.AddToScheme(scheme)) + require.NoError(t, edsdatadoghqv1alpha1.AddToScheme(scheme)) + + mgr, err := ctrl.NewManager(&rest.Config{}, manager.Options{ + Scheme: scheme, + LeaderElection: false, + Controller: ctrlconfig.Controller{ + SkipNameValidation: ptr.To(true), + }, + }) + require.NoError(t, err) + + reconciler := &DatadogAgentInternalReconciler{ + Client: mgr.GetClient(), + PlatformInfo: kubernetes.PlatformInfo{}, + Scheme: scheme, + Recorder: mgr.GetEventRecorderFor("datadogagentinternal-test"), + Options: test.options, + } + require.NoError(t, reconciler.SetupWithManager(mgr, noopMetricsForwardersManager{})) + require.NotNil(t, reconciler.internal) + }) + } +} + func TestResourceFallbackPodPredicate(t *testing.T) { predicate := resourceFallbackPodPredicate() assert.False(t, predicate.Create(event.CreateEvent{Object: &corev1.ConfigMap{}})) From 6b190d4f83a4cbc83b1c231739b6e09086197583 Mon Sep 17 00:00:00 2001 From: "ali.benabdallah" Date: Thu, 23 Jul 2026 15:39:20 +0200 Subject: [PATCH 10/16] Recover prepared Agent handoff reservations --- .../datadogagentinternal/prepared_rollout.go | 40 +- .../prepared_rollout_test.go | 354 +++++++++++++++++ .../resource_fallback_test.go | 356 ++++++++++++++++++ 3 files changed, 748 insertions(+), 2 deletions(-) diff --git a/internal/controller/datadogagentinternal/prepared_rollout.go b/internal/controller/datadogagentinternal/prepared_rollout.go index 1f619a1ca1..f675b5423e 100644 --- a/internal/controller/datadogagentinternal/prepared_rollout.go +++ b/internal/controller/datadogagentinternal/prepared_rollout.go @@ -390,12 +390,37 @@ func (r *Reconciler) reconcilePreparedHandoff(ctx context.Context, ddai *datadog if err != nil || budget <= 0 { return reconcile.Result{}, err } + candidates := preparedHandoffCandidates(liveDS, pods, currentRevision) + validReservations := make(map[string]struct{}, len(candidates)) + for _, candidate := range candidates { + if candidate.reserved { + validReservations[string(candidate.replacement.UID)] = struct{}{} + } + } + for i := range pods { + pod := &pods[i] + if pod.Spec.NodeName == "" || pod.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] != currentRevision || pod.Annotations[resourceFallbackOldPodAnnotation] == "" { + continue + } + if _, valid := validReservations[string(pod.UID)]; valid { + continue + } + return r.releasePreparedHandoffReservation(ctx, pod) + } + consumed := consumedFallbackBudget(liveDS, pods, currentRevision, time.Now()) - if consumed >= budget { + if consumed > budget { + for i := len(candidates) - 1; i >= 0; i-- { + if candidates[i].reserved { + return r.releasePreparedHandoffReservation(ctx, candidates[i].replacement) + } + } return reconcile.Result{}, nil } - candidates := preparedHandoffCandidates(liveDS, pods, currentRevision) for _, candidate := range candidates { + if !candidate.reserved && consumed >= budget { + continue + } if !candidate.reserved { base := candidate.replacement.DeepCopy() patched := candidate.replacement.DeepCopy() @@ -407,6 +432,7 @@ func (r *Reconciler) reconcilePreparedHandoff(ctx context.Context, ddai *datadog return reconcile.Result{}, fmt.Errorf("reserve prepared Agent handoff for Pod %s/%s: %w", patched.Namespace, patched.Name, err) } candidate.replacement = patched + consumed++ } liveCandidate, err := r.revalidatePreparedHandoff(ctx, liveDS, candidate, currentRevision) if err != nil { @@ -434,6 +460,16 @@ func (r *Reconciler) reconcilePreparedHandoff(ctx context.Context, ddai *datadog return reconcile.Result{}, nil } +func (r *Reconciler) releasePreparedHandoffReservation(ctx context.Context, replacement *corev1.Pod) (reconcile.Result, error) { + base := replacement.DeepCopy() + patched := replacement.DeepCopy() + delete(patched.Annotations, resourceFallbackOldPodAnnotation) + if err := r.client.Patch(ctx, patched, client.MergeFrom(base)); err != nil { + return reconcile.Result{}, fmt.Errorf("release prepared Agent handoff reservation for Pod %s/%s: %w", patched.Namespace, patched.Name, err) + } + return reconcile.Result{RequeueAfter: time.Second}, nil +} + func preparedHandoffCandidates(ds *appsv1.DaemonSet, pods []corev1.Pod, currentRevision string) []preparedHandoffCandidate { oldByNode := map[string]*corev1.Pod{} for i := range pods { diff --git a/internal/controller/datadogagentinternal/prepared_rollout_test.go b/internal/controller/datadogagentinternal/prepared_rollout_test.go index 136b29eb16..9311a3fde6 100644 --- a/internal/controller/datadogagentinternal/prepared_rollout_test.go +++ b/internal/controller/datadogagentinternal/prepared_rollout_test.go @@ -6,6 +6,7 @@ package datadogagentinternal import ( "context" + "errors" "strings" "testing" "time" @@ -20,6 +21,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/reconcile" apicommon "github.com/DataDog/datadog-operator/api/datadoghq/common" @@ -437,6 +439,358 @@ func TestReconcilePreparedHandoffReservesThenDeletesOldPod(t *testing.T) { assert.Equal(t, string(old.UID), updated.Annotations[resourceFallbackOldPodAnnotation]) } +func TestReconcilePreparedHandoffResumesReservedCandidateAtBudget(t *testing.T) { + fixture := newPreparedHandoffFixture(t) + fixture.replacement.Annotations = map[string]string{resourceFallbackOldPodAnnotation: string(fixture.old.UID)} + base := fixture.client(t, true, true, true) + r := &Reconciler{client: base, apiReader: base} + + result, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.Equal(t, time.Second, result.RequeueAfter) + err = base.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{}) + assert.True(t, apierrors.IsNotFound(err), "a restart after reserving the full budget must resume and delete the exact old Pod") +} + +func TestReconcilePreparedHandoffFindsReservedCandidateAfterUnreservedAtBudget(t *testing.T) { + fixture := newPreparedHandoffFixture(t) + fixture.old.Spec.NodeName = "node-b" + fixture.replacement.Spec.NodeName = "node-b" + fixture.replacement.Annotations = map[string]string{resourceFallbackOldPodAnnotation: string(fixture.old.UID)} + + oldA := fixture.old.DeepCopy() + oldA.Name = "old-a" + oldA.UID = "old-a-uid" + oldA.Spec.NodeName = "node-a" + replacementA := fixture.replacement.DeepCopy() + replacementA.Name = "new-a" + replacementA.UID = "new-a-uid" + replacementA.Spec.NodeName = "node-a" + replacementA.Annotations = nil + + base := fixture.client(t, true, true, true) + require.NoError(t, base.Create(context.Background(), oldA)) + require.NoError(t, base.Create(context.Background(), replacementA)) + r := &Reconciler{client: base, apiReader: base} + + result, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.Equal(t, time.Second, result.RequeueAfter) + err = base.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{}) + assert.True(t, apierrors.IsNotFound(err), "the reserved handoff must resume even when an unreserved node sorts first") + assertOldPodStillExists(t, base, oldA) +} + +func TestReconcilePreparedHandoffReleasesStaleReservation(t *testing.T) { + fixture := newPreparedHandoffFixture(t) + fixture.replacement.Annotations = map[string]string{resourceFallbackOldPodAnnotation: string(fixture.old.UID)} + fixture.replacement.Status.ContainerStatuses[0].RestartCount = 1 + base := fixture.client(t, true, true, true) + r := &Reconciler{client: base, apiReader: base} + + result, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.Equal(t, time.Second, result.RequeueAfter) + updated := &corev1.Pod{} + require.NoError(t, base.Get(context.Background(), client.ObjectKeyFromObject(fixture.replacement), updated)) + assert.Empty(t, updated.Annotations[resourceFallbackOldPodAnnotation], "an ineligible replacement must stop consuming rollout budget") + assertOldPodStillExists(t, base, fixture.old) +} + +func TestReconcilePreparedHandoffReleasesReservationAboveReducedBudget(t *testing.T) { + fixture := newPreparedHandoffFixture(t) + fixture.old.Spec.NodeName = "node-a" + fixture.replacement.Spec.NodeName = "node-a" + fixture.replacement.Annotations = map[string]string{resourceFallbackOldPodAnnotation: string(fixture.old.UID)} + + oldB := fixture.old.DeepCopy() + oldB.Name = "old-b" + oldB.UID = "old-b-uid" + oldB.Spec.NodeName = "node-b" + replacementB := fixture.replacement.DeepCopy() + replacementB.Name = "new-b" + replacementB.UID = "new-b-uid" + replacementB.Spec.NodeName = "node-b" + replacementB.Annotations = map[string]string{resourceFallbackOldPodAnnotation: string(oldB.UID)} + + base := fixture.client(t, true, true, true) + require.NoError(t, base.Create(context.Background(), oldB)) + require.NoError(t, base.Create(context.Background(), replacementB)) + r := &Reconciler{client: base, apiReader: base} + + result, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.Equal(t, time.Second, result.RequeueAfter) + updated := &corev1.Pod{} + require.NoError(t, base.Get(context.Background(), client.ObjectKeyFromObject(replacementB), updated)) + assert.Empty(t, updated.Annotations[resourceFallbackOldPodAnnotation], "one reservation must be released when the budget is reduced") + assertOldPodStillExists(t, base, fixture.old) + assertOldPodStillExists(t, base, oldB) +} + +func TestReconcilePreparedHandoffFailsClosed(t *testing.T) { + t.Run("API reader error", func(t *testing.T) { + fixture := newPreparedHandoffFixture(t) + base := fixture.client(t, true, true, true) + reader := interceptor.NewClient(base, interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return errors.New("read failed") + }, + }) + r := &Reconciler{client: base, apiReader: reader} + _, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.ErrorContains(t, err, "read failed") + }) + + t.Run("foreign DaemonSet", func(t *testing.T) { + fixture := newPreparedHandoffFixture(t) + fixture.ds.OwnerReferences[0].UID = "other-ddai" + base := fixture.client(t, true, true, true) + r := &Reconciler{client: base, apiReader: base} + result, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.Equal(t, reconcile.Result{}, result) + assertOldPodStillExists(t, base, fixture.old) + }) + + t.Run("missing current revision", func(t *testing.T) { + fixture := newPreparedHandoffFixture(t) + base := fixture.client(t, false, true, true) + r := &Reconciler{client: base, apiReader: base} + result, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.Equal(t, reconcile.Result{}, result) + assertOldPodStillExists(t, base, fixture.old) + }) + + t.Run("revision list error", func(t *testing.T) { + fixture := newPreparedHandoffFixture(t) + base := fixture.client(t, true, true, true) + reader := interceptor.NewClient(base, interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*appsv1.ControllerRevisionList); ok { + return errors.New("revision list failed") + } + return c.List(ctx, list, opts...) + }, + }) + r := &Reconciler{client: base, apiReader: reader} + _, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.ErrorContains(t, err, "revision list failed") + assertOldPodStillExists(t, base, fixture.old) + }) + + t.Run("Pod list error", func(t *testing.T) { + fixture := newPreparedHandoffFixture(t) + base := fixture.client(t, true, true, true) + reader := interceptor.NewClient(base, interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*corev1.PodList); ok { + return errors.New("Pod list failed") + } + return c.List(ctx, list, opts...) + }, + }) + r := &Reconciler{client: base, apiReader: reader} + _, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.ErrorContains(t, err, "Pod list failed") + assertOldPodStillExists(t, base, fixture.old) + }) + + t.Run("invalid budget", func(t *testing.T) { + fixture := newPreparedHandoffFixture(t) + base := fixture.client(t, true, true, true) + r := &Reconciler{client: base, apiReader: base} + _, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromString("invalid")) + require.Error(t, err) + assertOldPodStillExists(t, base, fixture.old) + }) + + t.Run("budget already consumed", func(t *testing.T) { + fixture := newPreparedHandoffFixture(t) + fixture.ds.Status.NumberUnavailable = 1 + base := fixture.client(t, true, true, true) + r := &Reconciler{client: base, apiReader: base} + result, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.Equal(t, reconcile.Result{}, result) + assertOldPodStillExists(t, base, fixture.old) + }) + + t.Run("mismatched reservation", func(t *testing.T) { + fixture := newPreparedHandoffFixture(t) + fixture.replacement.Annotations = map[string]string{resourceFallbackOldPodAnnotation: "another-old-pod"} + base := fixture.client(t, true, true, true) + r := &Reconciler{client: base, apiReader: base} + result, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.Equal(t, time.Second, result.RequeueAfter) + updated := &corev1.Pod{} + require.NoError(t, base.Get(context.Background(), client.ObjectKeyFromObject(fixture.replacement), updated)) + assert.Empty(t, updated.Annotations[resourceFallbackOldPodAnnotation]) + assertOldPodStillExists(t, base, fixture.old) + }) + + t.Run("reservation patch error", func(t *testing.T) { + fixture := newPreparedHandoffFixture(t) + base := fixture.client(t, true, true, true) + writer := interceptor.NewClient(base, interceptor.Funcs{ + Patch: func(context.Context, client.WithWatch, client.Object, client.Patch, ...client.PatchOption) error { + return errors.New("patch failed") + }, + }) + r := &Reconciler{client: writer, apiReader: base} + _, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.ErrorContains(t, err, "reserve prepared Agent handoff") + assertOldPodStillExists(t, base, fixture.old) + }) + + t.Run("old Pod delete error", func(t *testing.T) { + fixture := newPreparedHandoffFixture(t) + fixture.replacement.Annotations = map[string]string{resourceFallbackOldPodAnnotation: string(fixture.old.UID)} + base := fixture.client(t, true, true, true) + writer := interceptor.NewClient(base, interceptor.Funcs{ + Delete: func(context.Context, client.WithWatch, client.Object, ...client.DeleteOption) error { + return errors.New("delete failed") + }, + }) + r := &Reconciler{client: writer, apiReader: base} + _, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(2)) + require.ErrorContains(t, err, "delete old Agent Pod") + assertOldPodStillExists(t, base, fixture.old) + }) +} + +func TestRevalidatePreparedHandoffRejectsStaleState(t *testing.T) { + t.Run("API reader error", func(t *testing.T) { + fixture := newPreparedHandoffFixture(t) + base := fixture.client(t, true, true, true) + reader := interceptor.NewClient(base, interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return errors.New("read failed") + }, + }) + r := &Reconciler{apiReader: reader} + candidate := fixture.candidate(true) + got, err := r.revalidatePreparedHandoff(context.Background(), fixture.ds, candidate, "new-revision") + require.ErrorContains(t, err, "read failed") + assert.Nil(t, got) + }) + + tests := []struct { + name string + mutateExpected func(*appsv1.DaemonSet) + mutateCandidate func(*preparedHandoffCandidate) + mutateObjects func(*preparedHandoffFixture) + includeReplacement bool + includeOld bool + expectedRevision string + }{ + {name: "DaemonSet UID changed", mutateExpected: func(ds *appsv1.DaemonSet) { ds.UID = "stale-daemonset" }, includeReplacement: true, includeOld: true, expectedRevision: "new-revision"}, + {name: "revision changed", includeReplacement: true, includeOld: true, expectedRevision: "other-revision"}, + {name: "replacement disappeared", includeOld: true, expectedRevision: "new-revision"}, + {name: "old Pod disappeared", includeReplacement: true, expectedRevision: "new-revision"}, + {name: "replacement UID changed", mutateCandidate: func(candidate *preparedHandoffCandidate) { candidate.replacement.UID = "stale-replacement" }, includeReplacement: true, includeOld: true, expectedRevision: "new-revision"}, + {name: "reservation changed", mutateObjects: func(fixture *preparedHandoffFixture) { + fixture.replacement.Annotations = map[string]string{resourceFallbackOldPodAnnotation: "different-old-pod"} + }, includeReplacement: true, includeOld: true, expectedRevision: "new-revision"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fixture := newPreparedHandoffFixture(t) + fixture.replacement.Annotations = map[string]string{resourceFallbackOldPodAnnotation: string(fixture.old.UID)} + if test.mutateObjects != nil { + test.mutateObjects(fixture) + } + base := fixture.client(t, true, test.includeReplacement, test.includeOld) + r := &Reconciler{apiReader: base} + expected := fixture.ds.DeepCopy() + if test.mutateExpected != nil { + test.mutateExpected(expected) + } + candidate := fixture.candidate(true) + if test.mutateCandidate != nil { + test.mutateCandidate(&candidate) + } + got, err := r.revalidatePreparedHandoff(context.Background(), expected, candidate, test.expectedRevision) + require.NoError(t, err) + assert.Nil(t, got) + }) + } +} + +type preparedHandoffFixture struct { + scheme *runtime.Scheme + ddai *datadoghqv1alpha1.DatadogAgentInternal + ds *appsv1.DaemonSet + old *corev1.Pod + replacement *corev1.Pod + revision *appsv1.ControllerRevision +} + +func newPreparedHandoffFixture(t *testing.T) *preparedHandoffFixture { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, appsv1.AddToScheme(scheme)) + require.NoError(t, datadoghqv1alpha1.AddToScheme(scheme)) + + ddai := &datadoghqv1alpha1.DatadogAgentInternal{ObjectMeta: metav1.ObjectMeta{ + Name: "agent", Namespace: "datadog-agent", UID: "ddai-uid", Annotations: map[string]string{preparedRolloutAnnotation: "true"}, + }} + ds := preparedRolloutDaemonSet() + ds.UID = "daemonset-uid" + ds.Generation = 2 + ds.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: datadoghqv1alpha1.GroupVersion.String(), Kind: "DatadogAgentInternal", Name: ddai.Name, UID: ddai.UID, Controller: ptr.To(true), + }} + require.NoError(t, prepareAgentTemplate(ds, preparedRolloutPhaseStandby)) + require.True(t, configureResourceFallback(ds, intstr.FromInt(1))) + ds.Status = appsv1.DaemonSetStatus{ObservedGeneration: 2, DesiredNumberScheduled: 1} + + old := readyPod("old", "old-uid", "node-a", "old-revision", time.Now().Add(-time.Minute)) + old.Namespace = ds.Namespace + old.Labels["app"] = "agent" + old.OwnerReferences = []metav1.OwnerReference{daemonSetOwner(ds)} + replacement := preparedReplacementPod() + replacement.ObjectMeta = metav1.ObjectMeta{ + Name: "new", Namespace: ds.Namespace, UID: "new-uid", + Labels: map[string]string{"app": "agent", appsv1.DefaultDaemonSetUniqueLabelKey: "new-revision"}, + OwnerReferences: []metav1.OwnerReference{daemonSetOwner(ds)}, + } + replacement.Spec.NodeName = "node-a" + + return &preparedHandoffFixture{ + scheme: scheme, ddai: ddai, ds: ds, old: old, replacement: replacement, + revision: controllerRevisionForTemplate(t, ds, "new-revision"), + } +} + +func (f *preparedHandoffFixture) client(t *testing.T, includeRevision, includeReplacement, includeOld bool) client.WithWatch { + t.Helper() + objects := []client.Object{f.ddai, f.ds} + if includeRevision { + objects = append(objects, f.revision) + } + if includeReplacement { + objects = append(objects, f.replacement) + } + if includeOld { + objects = append(objects, f.old) + } + return fake.NewClientBuilder().WithScheme(f.scheme).WithObjects(objects...).Build() +} + +func (f *preparedHandoffFixture) candidate(reserved bool) preparedHandoffCandidate { + return preparedHandoffCandidate{replacement: f.replacement.DeepCopy(), old: f.old.DeepCopy(), nodeName: "node-a", reserved: reserved} +} + +func assertOldPodStillExists(t *testing.T, c client.Client, old *corev1.Pod) { + t.Helper() + require.NoError(t, c.Get(context.Background(), client.ObjectKeyFromObject(old), &corev1.Pod{})) +} + func preparedReplacementPod() *corev1.Pod { return &corev1.Pod{Status: corev1.PodStatus{ Phase: corev1.PodRunning, diff --git a/internal/controller/datadogagentinternal/resource_fallback_test.go b/internal/controller/datadogagentinternal/resource_fallback_test.go index 5e5b64353c..12928b4fb8 100644 --- a/internal/controller/datadogagentinternal/resource_fallback_test.go +++ b/internal/controller/datadogagentinternal/resource_fallback_test.go @@ -7,6 +7,7 @@ package datadogagentinternal import ( "context" "encoding/json" + "errors" "maps" "testing" "time" @@ -23,6 +24,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" datadoghqcommon "github.com/DataDog/datadog-operator/api/datadoghq/common" datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" @@ -126,6 +128,9 @@ func TestResourceOnlyUnschedulable(t *testing.T) { {name: "ephemeral storage is rejected", reason: corev1.PodReasonUnschedulable, message: "0/1 nodes are available: 1 Insufficient ephemeral-storage.", ok: false}, {name: "custom reason containing cpu text is rejected", reason: corev1.PodReasonUnschedulable, message: "0/1 nodes are available: 1 custom plugin: Insufficient cpu.", ok: false}, {name: "wrong condition reason", reason: "SchedulingGated", message: "0/1 nodes are available: 1 Insufficient cpu.", ok: false}, + {name: "empty primary reason", reason: corev1.PodReasonUnschedulable, message: "preemption:", ok: false}, + {name: "reason without count", reason: corev1.PodReasonUnschedulable, message: "0/1 nodes are available: cpu.", ok: false}, + {name: "non-numeric count", reason: corev1.PodReasonUnschedulable, message: "0/1 nodes are available: many Insufficient cpu.", ok: false}, } for _, tt := range tests { @@ -370,6 +375,11 @@ func TestExistingPodRequiredAntiAffinityCanRejectReplacement(t *testing.T) { assert.False(t, allowed, "wider topology terms fail closed without loading every Node's topology labels") } +func TestAffinityTermMaySelectNamespace(t *testing.T) { + assert.True(t, affinityTermMaySelectNamespace(&corev1.PodAffinityTerm{Namespaces: []string{"target"}}, "source", "target")) + assert.False(t, affinityTermMaySelectNamespace(&corev1.PodAffinityTerm{Namespaces: []string{"other"}}, "source", "target")) +} + func TestPodAffinityTermSelector(t *testing.T) { term := &corev1.PodAffinityTerm{ LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "agent"}}, @@ -426,6 +436,21 @@ func TestResourceFitAfterOldPodRemoval(t *testing.T) { staleMessage := node.DeepCopy() staleMessage.Status.Allocatable[corev1.ResourceCPU] = resource.MustParse("3") assert.False(t, resourceFitAfterOldPodRemoval(staleMessage, []corev1.Pod{*old}, replacement, old, resourceShortage{cpu: true}), "reported shortage must still be observable") + + claims := replacement.DeepCopy() + claims.Spec.ResourceClaims = []corev1.PodResourceClaim{{Name: "accelerator"}} + assert.False(t, resourceFitAfterOldPodRemoval(node, []corev1.Pod{*old}, claims, old, resourceShortage{cpu: true}), "dynamic resource claims are not modeled") + + assert.False(t, resourceFitAfterOldPodRemoval(node, nil, replacement, old, resourceShortage{cpu: true}), "the exact old Pod must still be present") + assert.False(t, resourceFitAfterOldPodRemoval(node, []corev1.Pod{*old}, replacement, old, resourceShortage{}), "a scheduler-reported CPU or memory shortage is required") + + podLimited := node.DeepCopy() + podLimited.Status.Allocatable[corev1.ResourcePods] = resource.MustParse("0") + assert.False(t, resourceFitAfterOldPodRemoval(podLimited, []corev1.Pod{*old}, replacement, old, resourceShortage{cpu: true}), "the replacement must fit the node Pod limit") + + finished := scheduledResourcePod("finished", "finished-uid", "node-a", "10", "10Gi") + finished.Status.Phase = corev1.PodSucceeded + assert.True(t, resourceFitAfterOldPodRemoval(node, []corev1.Pod{*old, *finished}, replacement, old, resourceShortage{cpu: true}), "terminal Pods do not consume scheduler capacity") } func TestSchedulerPodRequestsIncludesInitAndOverhead(t *testing.T) { @@ -571,6 +596,291 @@ func TestReconcileResourceFallbackRejectsForeignDaemonSet(t *testing.T) { require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{}), "foreign DaemonSet Pods must never be deleted") } +func TestReconcileResourceFallbackEarlyExitsAndErrors(t *testing.T) { + t.Run("uses cached client when API reader is absent", func(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + fixture.reconciler.apiReader = nil + result, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(0)) + require.NoError(t, err) + assert.Zero(t, result) + }) + + t.Run("missing daemonset", func(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + require.NoError(t, fixture.client.Delete(context.Background(), fixture.ds)) + result, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.Zero(t, result) + }) + + t.Run("daemonset read error", func(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + reader := interceptor.NewClient(fixture.client.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*appsv1.DaemonSet); ok { + return errors.New("read daemonset") + } + return c.Get(ctx, key, obj, opts...) + }, + }) + fixture.reconciler.apiReader = reader + _, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.ErrorContains(t, err, "get Agent DaemonSet") + }) + + t.Run("invalid and zero budgets", func(t *testing.T) { + for _, budget := range []intstr.IntOrString{intstr.FromString("invalid"), intstr.FromInt(0)} { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + result, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, budget) + if budget.Type == intstr.String { + require.ErrorContains(t, err, "resolve Agent resource fallback budget") + } else { + require.NoError(t, err) + assert.Zero(t, result) + } + } + }) + + t.Run("missing current revision", func(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + revisions := &appsv1.ControllerRevisionList{} + require.NoError(t, fixture.client.List(context.Background(), revisions)) + require.NotEmpty(t, revisions.Items) + require.NoError(t, fixture.client.Delete(context.Background(), &revisions.Items[0])) + result, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.Zero(t, result) + }) + + t.Run("revision and pod list errors", func(t *testing.T) { + for _, failPods := range []bool{false, true} { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + reader := interceptor.NewClient(fixture.client.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*appsv1.ControllerRevisionList); ok && !failPods { + return errors.New("list revisions") + } + if _, ok := list.(*corev1.PodList); ok && failPods { + return errors.New("list pods") + } + return c.List(ctx, list, opts...) + }, + }) + fixture.reconciler.apiReader = reader + _, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.Error(t, err) + } + }) + + t.Run("reservation patch error", func(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + writer := interceptor.NewClient(fixture.client.(client.WithWatch), interceptor.Funcs{ + Patch: func(context.Context, client.WithWatch, client.Object, client.Patch, ...client.PatchOption) error { + return errors.New("patch reservation") + }, + }) + fixture.reconciler.client = writer + _, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.ErrorContains(t, err, "reserve Agent resource fallback") + }) + + t.Run("old pod delete error", func(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + writer := interceptor.NewClient(fixture.client.(client.WithWatch), interceptor.Funcs{ + Delete: func(context.Context, client.WithWatch, client.Object, ...client.DeleteOption) error { + return errors.New("delete old pod") + }, + }) + fixture.reconciler.client = writer + _, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.ErrorContains(t, err, "delete old Agent Pod") + }) +} + +func TestFallbackBudgetWithinLimitFailsClosed(t *testing.T) { + t.Run("missing DaemonSet", func(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + require.NoError(t, fixture.client.Delete(context.Background(), fixture.ds)) + ok, err := fallbackBudgetWithinLimit(context.Background(), fixture.client, fixture.ds, intstr.FromInt(1), "new-revision") + require.NoError(t, err) + assert.False(t, ok) + }) + + t.Run("stale DaemonSet identity", func(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + expected := fixture.ds.DeepCopy() + expected.UID = "stale-uid" + ok, err := fallbackBudgetWithinLimit(context.Background(), fixture.client, expected, intstr.FromInt(1), "new-revision") + require.NoError(t, err) + assert.False(t, ok) + }) + + t.Run("revision changed", func(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + ok, err := fallbackBudgetWithinLimit(context.Background(), fixture.client, fixture.ds, intstr.FromInt(1), "other-revision") + require.NoError(t, err) + assert.False(t, ok) + }) + + t.Run("invalid budget", func(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + ok, err := fallbackBudgetWithinLimit(context.Background(), fixture.client, fixture.ds, intstr.FromString("invalid"), "new-revision") + require.Error(t, err) + assert.False(t, ok) + }) +} + +func TestControllerRevisionMatchesTemplateRejectsInvalidData(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + revisions := &appsv1.ControllerRevisionList{} + require.NoError(t, fixture.client.List(context.Background(), revisions)) + require.Len(t, revisions.Items, 1) + revision := revisions.Items[0].DeepCopy() + revision.Data.Raw = []byte("not-json") + + got, err := controllerRevisionMatchesTemplate(revision, &fixture.ds.Spec.Template) + require.Error(t, err) + assert.False(t, got) +} + +func TestFallbackCandidatesFailClosedAndSortReservations(t *testing.T) { + now := time.Now() + ds := &appsv1.DaemonSet{Spec: appsv1.DaemonSetSpec{MinReadySeconds: 0}} + pending := func(name, node, reservation string) corev1.Pod { + pod := pendingPodForNode(node) + pod.Name = name + pod.UID = types.UID(name + "-uid") + pod.Labels = map[string]string{appsv1.DefaultDaemonSetUniqueLabelKey: "new"} + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: corev1.PodReasonUnschedulable, Message: "0/1 nodes are available: 1 Insufficient cpu."}} + if reservation != "" { + pod.Annotations = map[string]string{resourceFallbackOldPodAnnotation: reservation} + } + return *pod + } + old := func(name, uid, node string) corev1.Pod { + pod := readyPod(name, uid, node, "old", now.Add(-time.Minute)) + return *pod + } + + noTarget := pending("no-target", "node-a", "") + noTarget.Spec.Affinity = nil + assert.Empty(t, fallbackCandidates(ds, []corev1.Pod{noTarget, old("old-a", "old-a-uid", "node-a")}, "new", now)) + assert.Empty(t, fallbackCandidates(ds, []corev1.Pod{pending("no-old", "node-a", "")}, "new", now)) + assert.Empty(t, fallbackCandidates(ds, []corev1.Pod{ + pending("two-old", "node-a", ""), old("old-a", "old-a-uid", "node-a"), old("old-b", "old-b-uid", "node-a"), + }, "new", now)) + assert.Empty(t, fallbackCandidates(ds, []corev1.Pod{ + pending("wrong-reservation", "node-a", "other-uid"), old("old-a", "old-a-uid", "node-a"), + }, "new", now)) + + pods := []corev1.Pod{ + pending("new-c", "node-c", ""), old("old-c", "old-c-uid", "node-c"), + pending("new-b", "node-b", ""), old("old-b", "old-b-uid", "node-b"), + pending("new-a", "node-a", "old-a-uid"), old("old-a", "old-a-uid", "node-a"), + } + candidates := fallbackCandidates(ds, pods, "new", now) + require.Len(t, candidates, 3) + assert.True(t, candidates[0].reserved) + assert.Equal(t, "node-a", candidates[0].nodeName) + assert.Equal(t, "node-b", candidates[1].nodeName) + assert.Equal(t, "node-c", candidates[2].nodeName) +} + +func TestRevalidateFallbackCandidateRejectsStaleStateAndReadErrors(t *testing.T) { + candidateFor := func(fixture fallbackTestFixture) fallbackCandidate { + return fallbackCandidate{pending: fixture.pending.DeepCopy(), old: fixture.old.DeepCopy(), nodeName: "node-a", shortage: resourceShortage{cpu: true}} + } + + t.Run("DaemonSet read error", func(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + reader := interceptor.NewClient(fixture.client.(client.WithWatch), interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return errors.New("read failed") + }, + }) + got, err := fixture.reconciler.revalidateFallbackCandidate(context.Background(), reader, fixture.ds, candidateFor(fixture), "new-revision", false) + require.ErrorContains(t, err, "read failed") + assert.Nil(t, got) + }) + + t.Run("stale DaemonSet", func(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + expected := fixture.ds.DeepCopy() + expected.UID = "stale-uid" + got, err := fixture.reconciler.revalidateFallbackCandidate(context.Background(), fixture.client, expected, candidateFor(fixture), "new-revision", false) + require.NoError(t, err) + assert.Nil(t, got) + }) + + t.Run("revision changed", func(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + got, err := fixture.reconciler.revalidateFallbackCandidate(context.Background(), fixture.client, fixture.ds, candidateFor(fixture), "other-revision", false) + require.NoError(t, err) + assert.Nil(t, got) + }) + + for _, objectName := range []string{"new", "old"} { + t.Run("missing "+objectName+" Pod", func(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + pod := fixture.pending + if objectName == "old" { + pod = fixture.old + } + require.NoError(t, fixture.client.Delete(context.Background(), pod)) + got, err := fixture.reconciler.revalidateFallbackCandidate(context.Background(), fixture.client, fixture.ds, candidateFor(fixture), "new-revision", false) + require.NoError(t, err) + assert.Nil(t, got) + }) + } + + t.Run("stale Pod UID", func(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + candidate := candidateFor(fixture) + candidate.pending.UID = "stale-pending" + got, err := fixture.reconciler.revalidateFallbackCandidate(context.Background(), fixture.client, fixture.ds, candidate, "new-revision", false) + require.NoError(t, err) + assert.Nil(t, got) + }) + + t.Run("reservation required", func(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + got, err := fixture.reconciler.revalidateFallbackCandidate(context.Background(), fixture.client, fixture.ds, candidateFor(fixture), "new-revision", true) + require.NoError(t, err) + assert.Nil(t, got) + }) + + t.Run("node disappeared", func(t *testing.T) { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + node := &corev1.Node{} + require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKey{Name: "node-a"}, node)) + require.NoError(t, fixture.client.Delete(context.Background(), node)) + got, err := fixture.reconciler.revalidateFallbackCandidate(context.Background(), fixture.client, fixture.ds, candidateFor(fixture), "new-revision", false) + require.NoError(t, err) + assert.Nil(t, got) + }) + + t.Run("Pod list errors", func(t *testing.T) { + for _, failCall := range []int{1, 2} { + fixture := newFallbackTestFixture(t, healthyNodeConditions()) + podLists := 0 + reader := interceptor.NewClient(fixture.client.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*corev1.PodList); ok { + podLists++ + if podLists == failCall { + return errors.New("list failed") + } + } + return c.List(ctx, list, opts...) + }, + }) + got, err := fixture.reconciler.revalidateFallbackCandidate(context.Background(), reader, fixture.ds, candidateFor(fixture), "new-revision", false) + require.ErrorContains(t, err, "list") + assert.Nil(t, got) + } + }) +} + func TestToleratesBlockingNodeTaints(t *testing.T) { taints := []corev1.Taint{ {Key: "dedicated", Value: "agents", Effect: corev1.TaintEffectNoSchedule}, @@ -588,6 +898,52 @@ func TestToleratesBlockingNodeTaints(t *testing.T) { assert.True(t, toleratesBlockingNodeTaints([]corev1.Toleration{{Operator: corev1.TolerationOpExists}}, taints)) } +func TestNodeReadyForResourceFallbackFailsClosed(t *testing.T) { + ready := &corev1.Node{Status: corev1.NodeStatus{Conditions: healthyNodeConditions()}} + assert.True(t, nodeReadyForResourceFallback(ready)) + + unschedulable := ready.DeepCopy() + unschedulable.Spec.Unschedulable = true + assert.False(t, nodeReadyForResourceFallback(unschedulable)) + + deleting := ready.DeepCopy() + deleting.DeletionTimestamp = &metav1.Time{Time: time.Now()} + assert.False(t, nodeReadyForResourceFallback(deleting)) + + networkUnavailable := ready.DeepCopy() + for i := range networkUnavailable.Status.Conditions { + if networkUnavailable.Status.Conditions[i].Type == corev1.NodeNetworkUnavailable { + networkUnavailable.Status.Conditions[i].Status = corev1.ConditionTrue + } + } + assert.False(t, nodeReadyForResourceFallback(networkUnavailable)) +} + +func TestResourceFallbackDaemonSetEligibleFailsClosed(t *testing.T) { + ds := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Generation: 1}, + Spec: appsv1.DaemonSetSpec{UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ + Type: appsv1.RollingUpdateDaemonSetStrategyType, + RollingUpdate: &appsv1.RollingUpdateDaemonSet{MaxSurge: ptr.To(intstr.FromInt(1))}, + }}, + Status: appsv1.DaemonSetStatus{ObservedGeneration: 1, DesiredNumberScheduled: 1}, + } + assert.True(t, resourceFallbackDaemonSetEligible(ds)) + + for _, mutate := range []func(*appsv1.DaemonSet){ + func(value *appsv1.DaemonSet) { value.Status.DesiredNumberScheduled = 0 }, + func(value *appsv1.DaemonSet) { value.Status.ObservedGeneration = 0 }, + func(value *appsv1.DaemonSet) { value.Spec.UpdateStrategy.Type = appsv1.OnDeleteDaemonSetStrategyType }, + func(value *appsv1.DaemonSet) { + value.Spec.UpdateStrategy.RollingUpdate.MaxSurge = ptr.To(intstr.FromInt(0)) + }, + } { + copy := ds.DeepCopy() + mutate(copy) + assert.False(t, resourceFallbackDaemonSetEligible(copy)) + } +} + type fallbackTestFixture struct { client client.Client reconciler *Reconciler From f2f8bacef5cf7e8c6d85857dc2b86c75f8c0adb8 Mon Sep 17 00:00:00 2001 From: "ali.benabdallah" Date: Fri, 24 Jul 2026 13:20:04 +0200 Subject: [PATCH 11/16] Prototype prepared Agent surge with capacity fallback --- docs/agent_host_network_surge_poc.md | 59 - docs/agent_zero_gap_rollouts_rfc.md | 164 --- docs/agent_zero_gap_rollouts_rfc_appendix.md | 970 --------------- experiments/openkruise-prepull/README.md | 378 ------ .../automatic-predownload.yaml | 43 - experiments/openkruise-prepull/base.yaml | 100 -- .../imagepulljob-activation-failure.yaml | 18 - .../openkruise-prepull/imagepulljob-bad.yaml | 18 - .../openkruise-prepull/imagepulljob-good.yaml | 18 - .../controller_reconcile_agent.go | 60 +- .../controller_reconcile_agent_test.go | 56 +- .../datadogagentinternal/prepared_rollout.go | 469 ++----- .../prepared_rollout_support.go | 229 ++++ .../prepared_rollout_test.go | 976 ++++----------- .../datadogagentinternal/resource_fallback.go | 649 +++------- .../resource_fallback_test.go | 1092 ++--------------- .../datadogagentinternal_controller.go | 23 +- .../datadogagentinternal_controller_test.go | 57 +- .../testutils/renderer/render_e2e_test.go | 28 +- 19 files changed, 949 insertions(+), 4458 deletions(-) delete mode 100644 docs/agent_host_network_surge_poc.md delete mode 100644 docs/agent_zero_gap_rollouts_rfc.md delete mode 100644 docs/agent_zero_gap_rollouts_rfc_appendix.md delete mode 100644 experiments/openkruise-prepull/README.md delete mode 100644 experiments/openkruise-prepull/automatic-predownload.yaml delete mode 100644 experiments/openkruise-prepull/base.yaml delete mode 100644 experiments/openkruise-prepull/imagepulljob-activation-failure.yaml delete mode 100644 experiments/openkruise-prepull/imagepulljob-bad.yaml delete mode 100644 experiments/openkruise-prepull/imagepulljob-good.yaml create mode 100644 internal/controller/datadogagentinternal/prepared_rollout_support.go diff --git a/docs/agent_host_network_surge_poc.md b/docs/agent_host_network_surge_poc.md deleted file mode 100644 index 8c93dd31e8..0000000000 --- a/docs/agent_host_network_surge_poc.md +++ /dev/null @@ -1,59 +0,0 @@ -# Prepared host-network Agent surge PoC - -See [RFC: Prepared per-node Agent rollouts](agent_zero_gap_rollouts_rfc.md) for -the lifecycle proposal, trade-offs, alternatives, and validation plan. - -This PoC keeps `override.nodeAgent.hostNetwork: true` while allowing a native -DaemonSet surge Pod to be scheduled beside the old Agent Pod. - -It is explicitly enabled with: - -```yaml -metadata: - annotations: - experimental.agent.datadoghq.com/host-network-surge-prepared: "true" -spec: - override: - nodeAgent: - hostNetwork: true - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 1 -``` - -The Operator first performs an `arm` rollout with `maxSurge: 0`. Once that -exact revision is fully Available it emits a `standby` template, removes -container port declarations, and changes the native DaemonSet strategy to -`maxUnavailable: 0`. Removing the declarations is necessary because Kubernetes -defaults every declared `containerPort` to the same `hostPort` for a -host-network Pod. The processes can still bind the node ports without PodSpec -port declarations. - -For DatadogAgentProfiles, the PoC narrows the standard Pod anti-affinity only -enough to let old and new revisions of the same DDA and profile overlap. Other -profiles and other DDA installations remain excluded. - -## Prepared-mode contract - -The first pilot fails closed unless the rendered Pod has exactly: - -- optimized Linux `agent` and `trace-agent` containers; -- standard `init-volume` and `init-config` init containers; -- `hostNetwork: true` and a RollingUpdate strategy; and -- no custom lifecycle hooks, reserved rollout paths, unsupported anti-affinity, - or custom commands. - -The Operator injects per-component node lock paths and Pod-private state paths, -bypasses `trace-loader`, and replaces network probes with state-file exec -probes. Startup accepts Prepared, waiting liveness accepts Prepared, and -post-activation liveness/readiness delegate to `agent health` or the local APM -listener. Once both containers have `Started=true`, the Operator annotates the -replacement with the old Pod UID and deletes that exact UID within the -`maxUnavailable` budget. The node locks keep the new processes asleep until the -old processes finish stopping. - -Emissary and all additional Agent containers must be disabled for this pilot. -CPU/memory fallback is independent and remains off unless -`experimental.agent.datadoghq.com/resource-fallback: "true"` is also set. diff --git a/docs/agent_zero_gap_rollouts_rfc.md b/docs/agent_zero_gap_rollouts_rfc.md deleted file mode 100644 index e76734e62e..0000000000 --- a/docs/agent_zero_gap_rollouts_rfc.md +++ /dev/null @@ -1,164 +0,0 @@ -# RFC: Prepared per-node Agent rollouts - -- Status: Draft -- Last updated: 2026-07-22 -- Owners: Agent and Datadog Operator - -## Decision - -Prototype native DaemonSet surge with an Agent **Prepared** state and an -Operator-controlled, one-node-at-a-time handoff. - -The replacement Pod is scheduled, its images and init containers complete, and -the real Agent processes start before the old Pod is terminated. Prepared -processes do not bind shared ports or UDS paths, start active collectors, mutate -shared log state, or acquire exclusive kernel resources. A failed pull, init, or -preparation therefore leaves the old Agent running unless the user-enabled -resource fallback has already deleted it. - -This phase removes the dominant pull/init/process-start delay. It does not claim -strict zero-gap handoff: releasing an old listener or collector and activating -its replacement leaves a smaller residual interval. If strict endpoint -continuity is required, the preferred extension is a stable node-local endpoint -holder or socket file-descriptor handoff. - -## Why this is needed - -The current DaemonSet rollout deletes the old Agent before scheduling, pulling, -initializing, and starting its replacement. A slow or failed pull can leave a -node without an Agent indefinitely. A slow system-probe teardown can also keep -the whole old Pod Terminating after sibling containers exit. - -Native `maxSurge` gives us create-before-delete, but production Agent Pods have -three overlap constraints: - -- With `hostNetwork: true`, Kubernetes defaults every declared `containerPort` - to the same `hostPort`; the scheduler rejects the second Pod. -- Both Pods mount the same UDS and host paths. A second active process can bind - or unlink the socket, duplicate log/check collection, or contend for kernel - resources. -- The scheduler accounts both Pods' CPU and memory requests. - -The design keeps `hostNetwork: true`. The Operator removes all regular and init -container `ports:` declarations from an explicitly compatible template; Agent -Prepared mode prevents runtime binds until activation. Shared hostPath mounting -itself is not the UDS conflict—the bind/unlink behavior is. - -## Proposed lifecycle - -1. Enabling the experiment first renders an **arm** revision with `maxSurge: 0`. - It installs the lock-aware Agent, state files, exec probes, and narrowed - profile anti-affinity through a conventional rollout. This bootstrap is - required because a legacy Agent does not hold the ownership lock. -2. Once the exact arm template is fully updated, Ready, and Available, the - Operator renders a **standby** revision. It keeps `hostNetwork: true`, strips - PodSpec port declarations, sets `maxUnavailable: 0`, and derives `maxSurge` - from the user's existing `maxUnavailable` budget. -3. A replacement constructs the real process graph, writes `prepared` to its - Pod-private state file, then waits on a per-component advisory `flock` in a - stable node hostPath. Startup accepts Prepared. While waiting, liveness - accepts Prepared; after activation, liveness and readiness delegate to the - component's real health mechanism so the state marker cannot mask a failure. -4. When every supported replacement container is Running and - `ContainerStatus.Started=true`, the Operator persists a token on that Pod and - UID-precondition deletes the old Pod. The old processes release their locks - only after stopping. -5. The replacements acquire the locks, start listeners and collectors, write - `active`, and become Ready. The token remains charged until that happens. - -The handoff gate is necessary because native `maxSurge` bounds preparation, not -termination and activation. Kubernetes frees a surge slot as soon as old-Pod -deletion begins. Without Active acknowledgement, many nodes could enter a long -handoff concurrently even with a small `maxSurge`. - -Prepared health must not use HTTP, TCP, or gRPC probes. Both host-network Pods use the -node IP, so a replacement probe can accidentally hit the old Agent and trigger -premature deletion. Exec probes must identify the process inside the container. - -The first cluster experiment deliberately supports only the optimized Linux -core and trace containers plus the standard `init-volume` and `init-config` -init containers. It bypasses `trace-loader`, which otherwise binds APM -endpoints before the trace process can wait. Emissary, process/system/security -agents, OTel, host profiler, and other sidecars are rejected or disabled until -their pre-Prepared side effects are audited. - -## Capacity policy - -The correct default is honest double requests during overlap. Assigning Agent -requests to a different "port-holder" Pod is not request transfer: scheduling, -QoS, CPU shares, memory protection, and eviction accounting apply to the holder's -cgroup, leaving the Agent under-requested. - -For constrained nodes, retain the PoC's resource-fit classification behind the -separate `experimental.agent.datadoghq.com/resource-fallback` opt-in. The -Operator may delete an old Pod only when the -replacement is Pending solely for CPU or memory, the node is healthy, supported -scheduling constraints are revalidated, removing that exact old Pod appears to -make the replacement fit, and the Operator first persists a handoff token. - -Normal approved-but-not-Active handoffs and fallback reserved/deleted-but-not- -Active nodes share one ledger and must not exceed the configured -`maxUnavailable`. A fallback token is reserved before the UID-preconditioned -delete and released only after the replacement reports Active. - -This fallback is best effort, not proof. Another Pod can take the freed capacity -before the Agent schedules, leaving no old Agent and a still-Pending replacement. -The replacement can also encounter a later pull or init failure after the old -Agent is gone. -Clusters requiring deterministic headroom can run a separate low-priority -Agent-sized placeholder on each node, at the cost of reserving overlap capacity -continuously. - -## Alternatives and trade-offs - -| Option | Benefit | Main limitation | Position | -|---|---|---|---| -| Native surge + Prepared Agent + handoff gate | Normal path preserves old through pull/init/start; no permanent data-plane hop | Agent/Operator lifecycle work, temporary double requests, residual activation gap | Lead prototype | -| Stable endpoint-holder DaemonSet | Preserves public ports and UDS inode; can drain/buffer | Tier-0 proxy, protocol/origin fidelity, its own upgrade problem; does not fence logs/system-probe | Prototype if continuity is required | -| Holder also owns Agent requests | Appears to avoid double requests | Requests protect the wrong Pod/cgroup | Reject | -| Per-node request placeholder | Honest deterministic surge headroom | Permanently reserves a second Agent-sized slot; global priority can consume it | Optional capacity policy | -| OpenKruise Standard surge | Partition, pause, node selection, PreDelete hooks | Same hostPort, UDS, request, and handoff-budget problems as native surge | Focused comparison | -| OpenKruise `InPlaceIfPossible` | One Pod; preserves unaffected containers and avoids duplicate requests | Changed container stops before replacement starts; failed pull can leave it down; unsupported changes recreate Pod | Complementary optimization | -| CSI | Can provide per-Pod/shared mount paths | Does not own or transfer a live UDS socket, bind ports, or select an active Agent | Not a solution alone | -| Service/CNI/eBPF indirection | Can move TCP/UDP endpoints off host networking | Changes UDP source/origin semantics and does not solve UDS or collectors | Environment-specific | - -OpenKruise should be evaluated as two separate experiments. Standard surge does -not remove the need for Prepared mode, and PreDelete alone does not hold a surge -slot through activation. In-place update is valuable for image-only or -single-container changes, but its image pre-download currently optimizes rather -than gates rollout, and Advanced DaemonSet has no fail-closed `InPlaceOnly` mode. -Single, non-comparable local v1.9.1 runs observed 7.845-second uncached and -4.511-second pre-pulled request gaps; a failed automatic pre-download still let -the selected Pod enter `ImagePullBackOff`. - -## Current PoC and required work - -The coordinated PoC now has three pieces under development: Agent process -locking/state, Operator arm-to-standby rendering and prepared handoff, and a -single-cluster ops override. The Operator fails closed on unsupported -containers, init containers, commands, lifecycle hooks, operating systems, -anti-affinity, and reserved volume paths. Ordinary native surge is unchanged -when the prepared-rollout annotation is absent, and resource fallback is -separately disabled by default. - -Before choosing a public API, validate the two-container pilot with numbered -metrics and traces, failed/slow pulls, failed preparation and activation, -Operator restart, and resource pressure with fallback both off and on. Expand -the allowlist only after logs/process/system-probe side effects are gated; the -full validation still includes a real two-minute system-probe teardown. Then -repeat the leading result on Linux KindVM and an experimental cluster. - -## Open decisions - -- Exact Prepared boundary and ownership groups for each Agent component. -- Authenticated transport for Prepared, handoff-approved, and Active status. -- Whether the residual activation interval is acceptable or requires a stable - endpoint holder. -- Whether constrained clusters prefer indefinite stall, explicit best-effort - fallback, or permanently reserved placeholder capacity. - -Detailed failure behavior, protocol notes, alternative analysis, validation -matrix, and primary sources are in the -[investigation appendix](agent_zero_gap_rollouts_rfc_appendix.md). The current -implementation notes are in the -[prepared host-network surge PoC](agent_host_network_surge_poc.md). diff --git a/docs/agent_zero_gap_rollouts_rfc_appendix.md b/docs/agent_zero_gap_rollouts_rfc_appendix.md deleted file mode 100644 index ecb96a1b9e..0000000000 --- a/docs/agent_zero_gap_rollouts_rfc_appendix.md +++ /dev/null @@ -1,970 +0,0 @@ -# Appendix: Prepared per-node Agent rollout investigation - -This appendix contains the detailed constraints, protocol proposal, alternative -analysis, failure modes, validation plan, and source material for -[RFC: Prepared per-node Agent rollouts](agent_zero_gap_rollouts_rfc.md). - -- Status: Draft -- Last updated: 2026-07-22 -- Owners: Agent and Datadog Operator -- Scope: Linux Kubernetes node Agent - -## Summary - -The Agent should normally prepare its replacement on each node before the old -Agent is terminated. A failed or slow image pull, initialization, or process -startup must leave the old Agent running unless the explicitly enabled resource -fallback has already deleted it. We must retain `hostNetwork: true` where it is -configured and support the host ports, Unix domain sockets (UDS), host paths, -log state, and kernel resources used by production installations. - -The leading design combines native DaemonSet surge with an Agent prepared mode: - -1. The Operator renders `maxSurge` and `maxUnavailable: 0`. -2. Kubernetes schedules the replacement beside the old Pod and completes image - pulls and init containers. -3. The new Agent processes start in a prepared state. They validate as much as - possible but do not bind shared ports, bind or unlink shared UDS paths, tail - logs, run checks, or acquire exclusive kernel resources. -4. Prepared processes report process health through container-local exec probes. - An Operator-controlled readiness gate admits only a bounded set of prepared - Pods to become Ready, allowing the DaemonSet controller to terminate those old - Pods. -5. Each old component releases a node-local ownership lock only after it has - stopped using its shared resources. Its prepared replacement acquires that - lock and activates. -6. The replacement reports Active. Only then may the Operator admit another - prepared replacement into the handoff budget. - -This design intentionally separates Kubernetes scheduling compatibility from -runtime ownership. Removing PodSpec port declarations lets two host-network Pods -be scheduled on one node; it does not let two processes bind the same address. -The prepared state is what prevents the runtime collision. - -The surge Pod normally needs a second full set of CPU and memory requests. When -that capacity is unavailable, an optional Operator fallback can classify the new -Pod as blocked only by node CPU or memory and estimate that deleting the old Pod -would make it fit. It then deletes the old Pod within the existing -`maxUnavailable` budget. This preserves rollout progress, but explicitly falls -back to the current availability behavior on those nodes and does not reserve -the freed capacity. - -This RFC recommends continuing the prepared native-surge prototype. A stable -per-node endpoint holder is a credible alternative if preserving UDS inodes or -long-lived connections is required. A different Pod must not hold the Agent's -resource requests: Kubernetes scheduling, QoS, and cgroup protection attach -those requests to the holder, not to the Agent. OpenKruise should be evaluated -both for Standard surge controls and, separately, for in-place updates, but -neither removes the need for Agent lifecycle work. - -## Status of the current PoC - -Three coordinated experimental branches implement the first testable slice. -The Agent branch adds a pre-start `flock` gate and atomic -`prepared`/`activating`/`active`/`stopped` state for core and trace, with -additional wiring under test for process and system-probe. The health listener -is moved out of graph construction so Prepared does not bind port 5555. - -The Operator branch: - -- uses `experimental.agent.datadoghq.com/host-network-surge-prepared=true` as - the explicit opt-in and otherwise leaves native surge unchanged; -- performs a conventional `arm` rollout before emitting a `standby` surge - revision, recording phase on the PodTemplate for restart safety; -- accepts only optimized Linux `agent` + `trace-agent` and the standard - `init-volume` + `init-config` init containers in the first pilot; -- keeps `hostNetwork: true`, narrows known profile anti-affinity while arming, - bypasses `trace-loader`, and strips port declarations only in standby; -- replaces every regular-container startup, liveness, and readiness probe with - an exec check of that container's private state file; -- treats `ContainerStatus.Started=true` as proof of Prepared, persists a token - on the replacement, and UID-precondition deletes the old Pod within the - existing `maxUnavailable` budget; and -- keeps CPU/memory resource fallback behind the separate, default-off - `experimental.agent.datadoghq.com/resource-fallback=true` annotation. - -The ops branch reserves one small experimental cluster, disables Emissary and -all containers outside the first allowlist, and caps the budget at one. Unit, -render, cache-transform, and controller tests cover the phase transition, -fail-closed rendering, local probes, token reservation, and old-UID deletion. -Real image builds and cluster data-plane validation remain outstanding; none of -these annotations are a supported production API. - -## Goals - -- Start replacement containers and Agent processes before terminating their old - counterparts. -- Leave the old Agent running indefinitely when the replacement image cannot be - pulled or the replacement cannot reach the prepared state. -- Preserve `hostNetwork: true` and existing node-facing port numbers. -- Support APM, DogStatsD, OTLP, UDS, logs, checks, system-probe, and other - enabled Agent containers without two active collectors on one node. -- Bound both preparation and post-delete activation concurrency with the - existing `maxUnavailable` policy rather than introducing a fixed percentage. -- Make capacity fallback explicit, conservative, observable, and optional. -- Support image, configuration, and resource changes. -- Fail closed when the Operator or Agent cannot prove that overlap is safe. - -## Non-goals - -- Image pre-pulling as the availability mechanism. It improves latency but does - not protect against pull, initialization, or process failures. -- Running two active Agents on one node. -- Eliminating the final release/acquire/bind activation interval in this phase. - Strict endpoint continuity requires a stable holder, socket activation, or - file-descriptor handoff. -- Hiding real CPU or memory use from the scheduler. -- Replacing `hostNetwork` or host-facing ingestion endpoints as a prerequisite. -- Designing the final public Operator API before the lifecycle works in a real - cluster. -- Guaranteeing gap-free handoff on a node where the configured policy permits - the resource fallback to delete the old Agent. - -## Terminology and invariants - -An Agent component is in one of these states: - -- **Starting**: the container or process has not completed safe initialization. -- **Prepared**: the process is running and healthy but owns no active node-wide - resources. -- **Active**: the process owns its required listeners, sockets, collectors, log - state, or kernel resources and performs its normal work. -- **Draining**: the process is terminating while it still owns some resources. - -The hard invariant is that at most one instance owns each node-wide resource -group. The phase-one availability target is to remove image pull, init, and -process startup from the downtime window. Lock release followed by acquisition -and endpoint binding still has a residual interval with no Active owner. The -interval must be measured and bounded operationally; strict zero-gap continuity -requires an endpoint holder, socket activation, or file-descriptor transfer. - -Prepared is not the same as Active. Kubernetes Pod Ready must temporarily mean -"safe for the old revision to terminate" for a surged replacement. Metrics and -status must expose Prepared and Active separately so users do not mistake two -Ready Pods for two active Agents. - -The resource fallback is a larger explicit exception. With fallback disabled, -an unschedulable replacement leaves the old Agent active and the rollout -stalled. With fallback enabled, availability on selected nodes may match the -current delete-first rollout while the number of handoffs remains within the -configured budget. The current PoC does not have a separate fallback opt-in and -must gain one before production evaluation. - -## Baseline behavior - -The current delete-first lifecycle is: - -1. Kubernetes marks the old Pod for deletion. -2. Its containers terminate. A slow system-probe teardown can keep the whole - Pod Terminating after sibling Agent containers have exited. -3. Kubernetes creates and schedules the replacement. -4. The node pulls images. -5. Init containers run and the Agent processes start. - -The telemetry gap contains scheduling, image pulling, initialization, and -process startup. A bad image can extend it indefinitely. Pre-pulling only -removes part of step 4 in the successful case. - -Native DaemonSet surge reverses the destructive part of this order. With -`maxSurge > 0`, Kubernetes creates a new Pod on a node that still has an old, -available Pod. It marks the old Pod for deletion only after the new Pod is Ready -for `minReadySeconds`. A Pending, image-pull-failing, or unready replacement -therefore leaves the old Pod running. - -Native surge does not bound the complete handoff. Once an old Pod has a deletion -timestamp, the DaemonSet controller stops counting that old/new pair against -`maxSurge`, even if the old containers are still draining and the replacement -has not become Active. A merely Prepared-and-Ready replacement can therefore -release a slot and let the controller advance across the cluster. An external -Active acknowledgement and admission gate are required to keep post-delete -handoffs within budget. - -If an old Pod becomes unavailable, the DaemonSet controller may create its -replacement without charging that node to the normal surge limit. Capacity and -observability must therefore tolerate more overlap than the healthy-rollout -`maxSurge` value during simultaneous failures. - -## Constraints - -### Host networking and ports - -`hostNetwork: true` puts both Pods in the same node network namespace. In -addition, Kubernetes defaults every declared `containerPort` to the same -`hostPort` for a host-network Pod. The scheduler's host-port filter then rejects -the second Pod before either process starts. - -Disabling only Datadog's `hostPortConfig` is insufficient because other -container port declarations may remain. The prepared template must omit the -entire `ports:` list from every regular and init container. PodSpec port entries -are metadata and scheduling declarations; a process can still bind the numeric -node port at runtime. - -Omitting the declarations solves only scheduling. If the prepared process calls -`bind(2)` on the production address while the old process is listening, one of -the processes still fails. The replacement must remain non-listening until it -owns the corresponding resource group. - -Network probes are also unsafe during overlap. Both host-network Pods have the -same Pod IP, so an HTTP or TCP probe directed at the replacement's numeric port -can reach the old Agent and falsely report the replacement healthy. Prepared -mode must use exec probes or another container-private health channel. Named -port references cannot survive removal of the `ports:` declarations and must -fail validation or be rewritten. - -Services and NetworkPolicies using numeric ports continue to describe the same -runtime ports. Any external object using a named target port must be detected -where possible and documented as incompatible. The Operator cannot discover -every externally managed object from a DaemonSet template. - -### Shared UDS paths - -Mounting the same hostPath into two Pods is allowed. The conflict is caused by -two processes binding, unlinking, or cleaning up the same socket pathname. -Prepared processes must not touch the production UDS path. - -A Unix socket connection refers to a kernel socket object, commonly described -by the filesystem device and inode associated with its bound pathname. If one -process unlinks the path and another binds the same path, the new pathname names -a new socket object. Existing connected clients remain attached to the old -object until it closes; new clients resolve the new object. Stream clients will -normally observe EOF or reset when the old server exits and must reconnect. -Connected datagram behavior and retry policy must be measured for supported -clients. - -Prepared surge has the same eventual rebind as the baseline, but it avoids -rebinding while the old server is live. It is therefore no worse than the -baseline for inode continuity, but it does not preserve the inode across the -handoff. A stable endpoint holder can preserve it. - -### Active collectors and host resources - -Ports and UDS are only the scheduler-visible conflicts. A second active Agent -can also duplicate checks, tail the same logs, race on registry or offset state, -attach duplicate eBPF programs, or contend for system-probe and security -resources. Each component needs an explicit prepared boundary; a generic -"ports are free" test is not enough. - -The ownership boundary should be per component or per tightly coupled resource -group. A Pod-wide gate would keep every replacement component asleep until the -slowest old container exits. With per-component gates, a new trace-agent can -activate after the old trace-agent exits while a new system-probe continues to -wait through a long old system-probe teardown. - -### Resource requests - -The scheduler sums the requests of the old and replacement Pods during surge. -The Kubernetes DaemonSet API explicitly warns that per-node DaemonSet resource -consumption can double. This is correct accounting: both Pods exist and both -can consume memory and CPU during preparation or a fault. - -Limits do not solve scheduling because scheduling is based on requests. Lowering -only the prepared Pod's request would under-account its real use and weaken its -QoS and eviction protection. Kubernetes does not provide a general primitive -that lends a CPU or memory request from one Pod to another. - -## Proposed design - -```mermaid -sequenceDiagram - participant DS as DaemonSet controller - participant New as Replacement Agent - participant Op as Operator handoff gate - participant Old as Old Agent - DS->>New: Create surge Pod - New->>New: Pull, init, start, reach Prepared - New-->>Op: Prepared acknowledgement - Op-->>New: Approve readiness when budget permits - New-->>DS: Pod Ready - DS->>Old: Delete old Pod - Old-->>New: Release component ownership locks - New->>New: Bind and activate components - New-->>Op: Active acknowledgements - Op->>Op: Release handoff token -``` - -### 1. Explicit capability gate - -Prepared surge remains opt-in until every enabled regular and init container -supports the protocol. The Operator must validate both the requested strategy -and image capabilities before it removes port declarations or relaxes -anti-affinity. - -A safe bootstrap is a one-time conventional rollout to an Agent version that -participates in ownership locks even when prepared surge is disabled. Prepared -surge can then be enabled for the next update. Custom images need an explicit -capability declaration or a runtime handshake; version-string guesses are not -sufficient. - -If validation fails, the Operator leaves the ordinary template unchanged, -reports a status condition and warning event, and does not claim surge safety. - -Every production init container must be enumerated and audited because it runs -while the old Agent is Active. An init container may prepare private files, but -it must not mutate shared UDS paths, shared log state, host permissions, kernel -state, or other node-wide resources. Exclusive work must move into the -post-ownership activation phase. Unknown or user-supplied init containers fail -closed unless they explicitly declare a reviewed overlap capability. - -### 2. Render a schedulable surge template - -For an eligible native DaemonSet, the Operator: - -- keeps `hostNetwork: true`; -- removes all container and init-container `ports:` declarations; -- rewrites every supported HTTP or TCP startup, readiness, liveness, and - lifecycle check as a container-local exec operation; -- rejects every network probe or hook it cannot rewrite, whether its port is - numeric or named; -- permits only the known DatadogAgentProfile anti-affinity transformation; -- emits `maxUnavailable: 0`; and -- uses the user's existing `maxUnavailable` value as both `maxSurge` and the - Operator handoff budget. The resource fallback uses the same ceiling only when - separately enabled. - -No constant rollout percentage is introduced. A value such as `1` stays `1`; a -percentage stays a percentage and is resolved against desired nodes using the -same rounding semantics as Kubernetes. - -### 3. Start Agent processes in prepared mode - -Image pulling and init containers finish before regular containers start. Each -enabled Agent binary then starts in a prepared mode that performs only safe -initialization. The exact boundary must be defined per component, but the -minimum contract is: - -- parse configuration and secrets; -- initialize internal state that is private to the container or revision; -- verify executable dependencies and permissions where this has no side - effects; -- expose process health through an exec-readable file or private mechanism; -- do not bind production TCP or UDP ports; -- do not bind, unlink, chmod, or clean up production UDS paths; -- do not start checks, log tailers, network collectors, or telemetry emission; -- do not acquire system-probe, eBPF, security, or other exclusive host - resources; and -- do not mutate shared log offset or registry state. - -This state should run the real Agent binary rather than a shell sleeping before -`exec`. The objective is to pay image, container, binary startup, configuration -parsing, and safe initialization costs before the old process exits. - -### 4. Report handoff readiness without network probes - -Every supported regular container gets a startup exec probe that accepts -`prepared`, `activating`, or `active`. Kubelet then records -`ContainerStatus.Started=true` for that exact container; a restart resets it. -Liveness accepts the waiting states so waiting is indefinite. Once `active`, -core probes delegate to `agent health` and trace probes connect to the -container-local APM listener. Readiness requires Active and the same real -health check, so a standby replacement is never a Service endpoint, cannot hit -the old host-network listener, and cannot remain healthy on a stale state file. - -The Operator maintains a handoff budget derived from the user's existing -`maxUnavailable` policy. When all expected containers are Running+Started, all -expected init containers exited zero, and the old Pod remains Available on the -same node, it annotates the replacement with the old UID. That persisted -reservation charges the budget across reconcile retries and Operator restarts. -After revalidation, the Operator deletes that exact UID. No Pod readiness gate -or status write is required. - -The token is charged until the replacement reaches Active/Ready. If activation -fails after deletion, no additional node is handed off once the budget is -exhausted. Native `maxSurge` may prepare another replacement, but the Operator -does not delete its old Pod. - -### 5. Fence activation with node-local ownership locks - -Each active component holds an exclusive kernel-backed lock on a stable file in -a shared hostPath. The lock file is never deleted or replaced. The component -itself, or a helper whose lifetime is coupled to the owned resources, holds the -file descriptor until listeners and other shared resources are released. - -The prepared replacement blocks on the same lock. Once it acquires ownership it -binds production endpoints, opens shared state, initializes host collectors, and -transitions to Active. A process crash releases the advisory lock with its file -descriptor. A plain marker file is insufficient because it can be left stale. - -Locks should be split where independent activation is safe, for example: - -- core Agent and DogStatsD endpoints; -- trace-agent endpoints; -- process-agent collectors; -- logs and their shared state; -- system-probe and its kernel resources; and -- security-agent resources. - -The exact grouping is an Agent design task. Lock acquisition alone is not a -license to unlink another process's UDS path: activation must first verify that -the path is absent or belongs to a dead owner and fail safely otherwise. - -### 6. Let the Operator initiate bounded termination - -A standby Pod cannot become Ready while the old process holds its lock, so -native DaemonSet readiness ordering alone would deadlock. The Operator uses the -Started statuses described above to reserve a handoff token and delete the old -Pod. The native DaemonSet controller still creates and node-targets the surge -replacement; the Operator owns only the prepared-to-termination edge. - -New components activate as their old counterparts release ownership. Their -readiness exec probes acknowledge Active after listeners, collectors, and -shared state are operational. An activation failure therefore keeps the -replacement NotReady and its token charged instead of allowing the rollout to -sweep the cluster. - -If the replacement never reaches Prepared because of an image, init, config, or -process problem, Kubernetes never deletes the old available Pod. The rollout -stalls, which is the required safe failure mode. - -### 7. Fall back only after a conservative resource-shortage estimate - -A surged replacement may remain Pending because the node cannot fit both Pods' -requests. If resource fallback is enabled, the Operator may delete the old Pod -only after all of these checks pass: - -- the pending Pod is the current DaemonSet revision and targets exactly one - node; -- the old Pod on that node is available and belongs to the previous revision; -- the scheduler reports only insufficient CPU and/or memory plus the expected - DaemonSet node-affinity mismatch on other nodes; -- there is no nomination or deletion already in progress; -- the node is Ready, schedulable, and free of memory, disk, PID, and network - pressure; -- the Pod uses a supported scheduler, volume, affinity, topology, host-port, and - resource shape; -- both directions of required Pod anti-affinity are satisfied; -- recomputing scheduler requests shows the replacement does not fit before, - but would fit in the observed snapshot after removing the exact old Pod; and -- a persistent token plus live-state recheck keeps normal approved-but-not- - Active handoffs and fallback reserved/deleted-but-not-Active nodes within one - unified configured budget. - -The fallback token is persisted before the UID-preconditioned delete and is -released only after the replacement reports Active. - -The delete uses a UID precondition. Unknown Pod-declared constraints and -unrecognized scheduler reasons fail closed. Cluster-specific plugins installed -under the default scheduler name are not discoverable through the Pod API; the -feature therefore also needs a scheduler configuration allowlist or an explicit -operational compatibility requirement. A warning event identifies the node, old -Pod, and replacement. - -This is not an atomic capacity reservation. After the checks and old-Pod delete, -another workload or nominated Pod can consume the freed capacity before the -replacement binds. The Agent may then remain Pending with no old Agent, possibly -indefinitely. It may also encounter an image pull or init failure only after the -old Pod has been deleted. Priority discipline, a preemptible placeholder, a scheduler -reservation, or direct binding would be required to close that race. Until one -is selected, fallback is a best-effort progress mechanism and must be explicitly -opted into with this failure mode visible to users. - -This fallback must never initially trigger for `ImagePullBackOff`, failed -readiness, bad configuration, host-port conflicts, disk or PID pressure, taints, -unknown affinity, or generic scheduling errors. Those failures leave the old -Agent running unless a prior resource fallback already deleted it. - -## Failure behavior - -| Failure | Expected behavior | -|---|---| -| Slow or failed image pull | Old Agent remains Active indefinitely on the normal path; after an explicit resource fallback delete, no old Agent remains. | -| Init-container failure | Old Agent remains Active indefinitely on the normal path; after an explicit resource fallback delete, no old Agent remains. | -| Agent cannot reach Prepared | Old Agent remains Active indefinitely. | -| Replacement is Prepared, old termination is slow | Prepared components wait; each activates only after its old counterpart releases ownership. | -| Normal ownership handoff | Pull, init, and process startup are already complete, but release/acquire/bind leaves a measured residual interval with no Active owner. | -| Activation fails after old exit | That node is unavailable; its handoff token remains consumed so additional nodes are not admitted. | -| Node lacks overlap CPU or memory, fallback disabled | Old Agent remains Active and rollout stalls. | -| Node lacks overlap CPU or memory, fallback enabled | Operator may delete the old Pod after a conservative fit estimate and within budget; another workload can still win the freed capacity and extend downtime. | -| Runtime port bind fails after ownership | Component remains unhealthy, does not unlink an unknown UDS, and surfaces an activation error. | -| Operator loses API connectivity | Existing Pods keep their current state; node-local ownership does not depend on a timely Operator reconcile. | -| Node or kubelet fails | Surge cannot guarantee node-local telemetry; ordinary Kubernetes node failure behavior applies. | - -## Trade-offs of the leading design - -### Advantages - -- Uses the upstream DaemonSet controller for create-before-delete ordering. -- Keeps the old Agent through pull, initialization, and prepared-process - failures. -- Retains `hostNetwork: true` and existing runtime port numbers. -- Does not add a permanent proxy to every telemetry path. -- Adds no cluster-wide workload controller dependency. -- Preserves honest per-Pod resource requests. -- Can activate components independently during a slow multi-container teardown. -- Handoff admission and fallback deletion counts use the policy users already - understand. - -### Costs and risks - -- Requires coordinated changes across multiple Agent binaries and a new - Operator handoff coordinator. -- Temporarily needs approximately two Pods' requests on surged nodes. -- Pod Ready has prepared semantics during handoff and needs separate Active - observability. -- Native surge alone does not bound post-delete activation; a missing or faulty - Active acknowledgement could stall or over-advance the rollout. -- Removing port declarations can break named target-port consumers the Operator - cannot discover. -- Every shared host resource must be audited; an omitted side effect can create - duplicate telemetry or host contention. -- Ownership lock bootstrap and mixed Agent versions require a deliberate - compatibility rollout. -- The resource fallback deliberately gives up zero-gap behavior on constrained - nodes and cannot reserve the capacity it predicts will be freed. -- Release/acquire/bind still leaves a residual no-owner interval during a normal - handoff. -- Socket inode continuity is no better than the baseline once the old socket is - closed and rebound. - -## Alternatives - -### A. Stable per-node endpoint holder - -A small, rarely updated DaemonSet can own the public host-network ports and UDS -paths. Agent Pods listen on generation-specific private ports or socket paths. -The holder health-checks backends, atomically selects the active generation, and -optionally drains old connections. - -This is the strongest endpoint abstraction: - -- the public UDS pathname and inode can remain stable across Agent updates; -- public ports never move between Agent processes; -- TCP and HTTP connections can be drained; -- bounded UDP or stream buffering can hide a short backend restart; and -- Agents may use pod networking or unique host-network backend ports. - -It is also a new node-wide data-plane dependency: - -- a holder failure interrupts metrics and traces even when the Agent is healthy; -- the holder has its own difficult upgrade problem because a second holder - cannot bind the public endpoints; -- DogStatsD UDP and datagram UDS forwarding must preserve packet boundaries and - sender credentials or origin detection can change; -- TCP keep-alive and gRPC connections remain pinned to an old backend until - drained or reset; -- queues require explicit bounds, backpressure, and drop telemetry; -- logs, checks, system-probe, and kernel ownership are not proxyable and still - need prepared/active fencing; and -- adoption requires a one-time coordinated migration of endpoints from the - Agent to the holder. - -The holder is worth prototyping if UDS inode continuity, connection drain, or -brief ingestion buffering becomes a hard requirement. It is not required merely -to schedule the sleeping replacement. - -### B. A holder that also owns the Agent's resource requests - -This variant gives the stable holder an Agent-sized CPU and memory request and -gives Agent Pods very small or zero requests, hoping old and new Agents can share -the holder's reservation. - -Reject this design. Kubernetes does not transfer requests across Pods. CPU -shares, memory protection, QoS class, quota attribution, eviction priority, and -capacity accounting apply to the holder's cgroup. The active Agent remains -under-requested, and two Agent Pods can consume memory simultaneously despite -only one being represented to the scheduler. Limits cap consumption but do not -repair placement or QoS semantics. - -Endpoint ownership and expendable capacity reservation are also incompatible -lifecycles. A Pod that must be deleted or preempted to release capacity cannot -simultaneously provide stable ports and UDS. - -If an endpoint holder and capacity reservation are both tested, they must be -separate workloads. - -### C. Low-priority per-node surge placeholder - -A separate low-priority DaemonSet can reserve one Agent-sized slot on every -node. Agent Pods have a higher PriorityClass. When a surge Pod needs capacity, -the scheduler preempts only the placeholder; after the old Agent exits, the -placeholder returns. - -This preserves honest Agent requests and uses native scheduling. It also -permanently withholds enough allocatable capacity for two Agents per node, which -is the same capacity cost as guaranteeing every surge will fit. It cannot create -physical memory. Kubernetes priority is global, so another higher-priority -workload can evict the placeholder and consume the intended slot. Victim grace -period also adds delay. Deterministic headroom therefore requires cluster-wide -priority discipline and a short placeholder termination grace. This is an -operational policy option for clusters willing to pay for reserved headroom, not -a general default. - -### D. Start small, then resize the replacement Pod - -A custom controller could create the Prepared replacement with small CPU and -memory requests. After the old Pod exits, it could use in-place Pod resize to -raise the new Pod to the normal requests before activation. - -This avoids permanent headroom and reduces scheduler overlap, but it is not an -atomic request transfer. Resize can remain `Deferred` or become `Infeasible`; -QoS class cannot change; only CPU and memory are supported; and static CPU or -memory-manager policies, Windows, and Kubernetes version or feature gates add -constraints. Until the resize succeeds, the prepared process is under-protected -and can consume more than the scheduler accounted. DaemonSet template drift and -rollback semantics also require a custom controller. Keep this as an experiment, -not an availability foundation. - -### E. Dynamic Resource Allocation or a custom ResourceClaim - -Dynamic Resource Allocation assigns devices or other driver-managed resources. -A custom driver could serialize a synthetic "node Agent endpoint" claim, but it -would not proxy a port, preserve a Unix socket, pass a socket file descriptor, or -reserve generic CPU and memory for another Pod. Exclusive allocation would also -block the desired Prepared Pod from being scheduled concurrently. The API and -driver footprint do not buy the lifecycle primitive this design needs. - -### F. OpenKruise Advanced DaemonSet: Standard surge - -OpenKruise Standard rolling update with `maxSurge` also creates the replacement -before deleting the old Pod and supports `minReadySeconds`, partition, node -selection, pause, and PreDelete lifecycle hooks. - -It has the same fundamental overlap constraints as native surge: - -- Kubernetes still defaults and schedules host ports; -- both Pods still share host-network and UDS namespaces; -- requests still double; -- a sleeping replacement still needs a Prepared readiness contract; and -- active collectors still need Agent fencing. - -OpenKruise has no built-in resource-unschedulable fallback. Its richer rollout -and PreDelete controls may simplify an explicit handoff, so it deserves a focused -prototype, but it is an added CRD/controller/webhook/node-daemon dependency and -is not by itself the availability solution. - -PreDelete is not itself a handoff-budget primitive. A Pod in OpenKruise's -pre-deleting state can stop consuming its surge slot before the hook completes, -just as a native deleting Pod does. Bounding activation still needs an external -coordinator, or carefully controlled pause and partition progression tied to an -Active acknowledgement. - -### G. OpenKruise Advanced DaemonSet: `InPlaceIfPossible` - -An in-place update preserves the Pod UID, node, network namespace, and mounted -volumes. It restarts changed containers while unaffected containers continue. -This avoids a second Pod, rescheduling, duplicate requests, and complete Pod -teardown. It is attractive when one Agent container changes or a long -system-probe teardown should not block an unrelated component update. - -It does not overlap old and new instances of a changed container. Kubelet stops -that container before pulling and starting its new image, so a failed image pull -can leave the component down. OpenKruise image pre-download currently improves -the common case but does not gate the rollout on successful pulls. - -Supported in-place changes are narrower than arbitrary Pod-template changes. -Unsupported changes under `InPlaceIfPossible` fall back to Pod recreation. -Advanced DaemonSet does not provide a fail-closed `InPlaceOnly` mode. CPU and -memory resize support also depends on Kubernetes feature support. - -Treat in-place update as a complementary optimization, not the primitive that -satisfies the failed-pull invariant. Useful upstream contributions would be an -Advanced DaemonSet `InPlaceOnly` policy, a real image-pre-download success gate, -and explicit prepared/activation lifecycle support. - -#### Local OpenKruise v1.9.1 result - -A Kubernetes v1.36.1 Kind experiment measured one image-only update with an -uncached image and one with a standalone, successfully completed ImagePullJob. -The uncached update had a 7.845-second success-to-success request gap, including -a kubelet-reported 2.559-second pull. A different, pre-pulled image had a -4.511-second gap and was reported already present by kubelet, but retained the -container restart and readiness gap. These single runs are not a causal timing -comparison. - -A failed standalone ImagePullJob left the old Pod, container ID, restart count, -image, and traffic unchanged because the DaemonSet was not mutated. In contrast, -successful pre-pull of an incompatible image was followed by exit 127 and -`CrashLoopBackOff` after activation. Adding an environment variable caused -`InPlaceIfPossible` to recreate the Pod under a new UID. - -OpenKruise's automatic AdvancedDaemonSet pre-download is an alpha feature gate -that defaults off. With it enabled on a two-worker test, the controller created -an owned ImagePullJob and started an in-place update without waiting: the old -container exited two seconds after the job started while that job was still -active, and the Pod entered `ImagePullBackOff`. Exact commands, identities, and -timestamps are in `experiments/openkruise-prepull/README.md`. - -### H. Endpoint holder sidecar plus OpenKruise in-place update - -An endpoint holder in the same Pod can remain running while OpenKruise updates -only Agent containers. With Pod-level resources, the holder and Agent can share -one correctly scoped Pod budget, and the holder can preserve listeners during -eligible image-only updates. - -This hybrid avoids the cross-Pod request problem but does not preserve endpoints -when an unsupported change recreates the Pod. It also still has no overlapping -old and new Agent process, so the holder needs buffering to mask process startup -and cannot cover logs or kernel collectors. It is a promising optimization for -specific update classes, not a universal rollout model. - -### I. CSI for the shared UDS - -A CSI driver can provision or mount a shared directory, choose per-Pod backing -paths, and provide mount lifecycle hooks. It cannot preserve or transfer a live -socket object, bind host-network ports, select an active Agent backend, or stop a -process from unlinking the shared pathname. - -A CSI node plugin could itself own and proxy the public UDS, but then it is the -stable endpoint-holder design packaged as storage infrastructure. CSI alone does -not solve the socket ownership problem and is unnecessary for two Pods to mount -the existing hostPath. - -### J. Node-local Service, CNI, or eBPF endpoint indirection - -Agents can use pod networking and a node-local Service, NodePort, CNI redirect, -or eBPF program to expose stable node endpoints. This removes Pod host-port -reservations and can select an active backend. - -The approach changes networking and attribution semantics. UDP source identity, -DogStatsD origin detection, host reachability, NetworkPolicy behavior, and -support across customer CNIs must be validated. It does not solve UDS, logs, or -kernel ownership. It may be appropriate for a controlled internal environment -but is a larger compatibility change than prepared host-network surge. - -### K. `SO_REUSEPORT` - -Linux `SO_REUSEPORT` can allow multiple processes to bind the same TCP or UDP -address after scheduler reservations are removed. The kernel distributes flows -or datagrams between listeners; it does not provide the required active/passive -ownership. A privileged eBPF reuseport selector could implement selection, but -that is another endpoint-indirection data plane, is Linux-specific, and does -nothing for filesystem UDS paths, logs, or kernel collectors. It is not a -general handoff mechanism. - -### L. Two DaemonSets or a custom per-node rollout controller - -The Operator can manage old and new DaemonSets, or a new controller can create -one replacement Pod per selected node and coordinate explicit handoffs. -This offers full state-machine control, including scheduling timeouts and -resource fallback. - -It recreates substantial logic already present in the native DaemonSet -controller, increases API objects and reconciliation state, and still requires -the same prepared Agent behavior for ports, UDS, and active collectors. It is -justified only if native readiness-driven ordering cannot express the required -handoff. - -### M. Image pre-pull only - -An ImagePullJob, pre-pull DaemonSet, or runtime cache warmer reduces successful -rollout time. It does not keep the old process through initialization or startup, -does not guarantee that every layer remains present, and does not solve a bad -config or failed process. Keep it as an optional performance optimization. - -### N. In-Agent supervisor, socket activation, or file-descriptor transfer - -A long-lived supervisor can own listeners, download or select versioned Agent -binaries, launch a new child, and pass file descriptors with socket activation. -This can provide the most exact handoff and preserve socket objects. - -It moves image and process lifecycle outside ordinary Kubernetes container -semantics, complicating supply-chain policy, rollback, observability, and -resource isolation. A stable holder that runs as an explicit container is easier -to reason about. Keep this as a long-term Agent architecture option. - -## Decision matrix - -| Option | Old survives failed pull | New process starts first | Stable public endpoints | Request model | Change coverage | Complexity | -|---|---:|---:|---:|---|---|---| -| Current delete-first | No | No | No | One Pod | All template changes | Low | -| Image pre-pull only | Only before rollout | No | No | One Pod | Images | Low | -| Native surge + prepared Agent + handoff gate | Normal path | Yes | Same address/path, rebound at activation | Honest; temporarily two Pods | All Pod replacements | High | -| Stable endpoint holder + prepared Agents | Yes | Yes | Yes while holder lives | Honest; two Agents plus holder | All Agent replacements | High | -| Holder also owns Agent requests | Superficially | Yes | Yes | Incorrect cross-Pod accounting | All Agent replacements | Reject | -| Per-node placeholder + native surge | Yes | Yes | Same as prepared surge | Honest; permanently reserves overlap capacity | All Pod replacements | Medium/high cost | -| Small request then in-place resize | Conditional on resize | Yes | Same as prepared surge | Temporarily under-requested | CPU/memory and version dependent | High/experimental | -| OpenKruise Standard + prepared Agent | Yes | Yes | Same as native surge | Honest; temporarily two Pods | All Pod replacements | High dependency cost | -| OpenKruise in-place | No pull guarantee | No for changed container | Pod namespace persists | Honest; one Pod | Supported fields only; otherwise recreate | Medium/high | -| CSI alone | No | No | No | Unchanged | UDS mount lifecycle only | No useful solution alone | -| `SO_REUSEPORT` | Conditional | Yes | TCP/UDP only | Honest; temporarily two Pods | Does not provide active/passive handoff | Reject alone | -| Custom rollout controller + prepared Agent | Yes | Yes | Same as selected endpoint design | Honest; temporarily two Pods | All controlled changes | Very high | - -## Recommendation - -Continue with native DaemonSet surge, Agent prepared mode, and an -Active-acknowledged Operator handoff gate as the leading design because it -directly addresses the dominant delay without adding a permanent data-plane hop. -The small scheduler-compatibility PoC is implemented; the lifecycle and -coordinator are substantial, unimplemented work. - -Retain the current resource-fit classification logic, but place deletion behind -a separate explicit opt-in. Treat it as a best-effort, non-zero-gap escape hatch -for constrained nodes until capacity can be reserved atomically. - -Prototype two alternatives in parallel at small scope: - -1. OpenKruise `InPlaceIfPossible` for image-only and selected-container updates, - measuring the remaining restart gap and failed-pull behavior. -2. A minimal endpoint holder for DogStatsD UDP, APM TCP/HTTP, and stream/datagram - UDS, measuring origin metadata, connection drain, buffering, and inode - continuity. - -Do not move Agent requests to a different holder Pod. Evaluate the placeholder -DaemonSet only as an opt-in capacity policy for clusters willing to reserve -surge headroom permanently. - -## Security and operability - -Removing PodSpec port declarations does not reduce the privileges of a -host-network container. It only removes scheduler-visible reservations and API -metadata. Existing host-network, hostPath, kernel, and packet-capture risks -remain and should be documented independently. - -Prepared mode should reduce privileges before activation where practical, but a -single container cannot generally gain new Linux capabilities after startup. -The design must therefore treat the prepared process as privileged even while it -is sleeping and minimize its side effects. - -Ownership files require a root-owned host directory and stable permissions. A -lock key must identify the actual node-wide resource, such as protocol, address, -port, canonical UDS path, or kernel facility. Two installations cannot use -different lock keys to claim the same endpoint. Installation identity remains -useful for authorization and diagnostics, but not for weakening mutual -exclusion. Processes must not follow untrusted symlinks or replace lock files. -Endpoint holders require equivalent or greater hardening because they receive -all node-local telemetry and may preserve sender credentials. - -Mixed versions, rollback, node reboot, force deletion, kubelet restart, and -container-runtime cleanup need explicit tests. The safest response to an unknown -owner is to remain Prepared and report a blocking condition. - -## Observability - -The Operator and Agent should expose, per node and component: - -- rollout revision and old/replacement Pod UIDs; -- Starting, Prepared, Active, Draining, and fallback states; -- time spent pulling, initializing, prepared, waiting for ownership, activating, - and draining; -- ownership acquisition and release events; -- port or UDS bind failures and observed socket inode generation; -- resource-fallback candidates, reservations, deletions, and rejected reasons; -- active generation selected by an endpoint holder, if used; -- packets, requests, bytes, connections, queue depth, drops, and metadata loss - through a holder; and -- a per-node owner gauge that always alerts on multiple active owners and records - the duration of every zero-owner interval against the selected availability - policy. - -Pod Ready alone is not sufficient rollout telemetry. - -## Validation plan - -### Fast lab - -Use a two-worker Linux Kind cluster or KindVM and short teardown delays while -iterating. Docker Desktop is no longer available on the current workstation, so -local validation requires another container runtime or a remote Linux lab. - -Establish these baselines: - -1. Measure the current delete-first gap. -2. Show native surge works without host-network port reservations. -3. Show ordinary production `hostNetwork` plus declared ports blocks the second - Pod. -4. Show the prepared rendered template schedules two Pods while retaining - `hostNetwork: true`. -5. Reproduce a false-positive replacement probe against the old Agent's numeric - host-network health port, then show exec probes remove the alias. -6. Audit and exercise every production init container while the old Pod remains - Active. -7. Show a replacement that binds or unlinks production endpoints before - activation fails the safety tests. - -### Hypothesis-driven prototypes - -Run only the tests needed for the current question: - -- prepared Agent processes report Prepared without binding production - endpoints, while the unapproved Pod remains NotReady; -- the handoff coordinator approves no more than its budget and does not release - a token until all replacement components acknowledge Active; -- a long Terminating Pod cannot let the rollout accumulate unbounded handoffs; -- exec probes cannot accidentally interrogate the old host-network process; -- per-component locks allow trace/core activation while old system-probe still - drains; -- the Operator classifies only CPU or memory shortage, and a competing Pod test - demonstrates the non-atomic fallback race; -- the minimal endpoint holder preserves or intentionally translates source - metadata; and -- OpenKruise Standard and in-place strategies exhibit the documented failure - behavior. - -### Final validation for the leading design - -Use production Agent configuration and shared host resources. Generate numbered -signals so omissions and duplicates are visible: - -- DogStatsD metrics over UDP and UDS; -- APM traces over TCP/HTTP and supported UDS transports; -- OTLP gRPC and HTTP traffic; -- numbered log lines with restart and rotation cases; -- checks and process/network/security telemetry relevant to enabled components; - and -- long-lived and reconnecting socket clients. - -Exercise: - -- slow and failed image pulls; -- bad image references; -- init, configuration, startup, readiness, and activation failures; -- insufficient CPU, memory, pod count, and disk space; -- fallback enabled and disabled; -- image-only, configuration, resource, and mixed updates; -- Operator and API-server interruption; -- kubelet/container-runtime restart and Pod force deletion; -- rollback and mixed prepared-capable versions; and -- one real two-minute system-probe teardown while sibling containers exit. - -Pass phase one only if multiple Active owners are never observed, handoff and -fallback concurrency remain within their budgets, and every zero-owner interval -is attributable to the measured activation boundary or the explicitly enabled -fallback—not image pull, init, or prepared-process startup. Report numbered -metric, log, and trace omissions or duplicates rather than hiding them behind a -binary pass result. Strict zero-gap acceptance requires the endpoint-holder or -socket-handoff variant to demonstrate no zero-owner interval. Repeat the result -on Linux KindVM and an experimental cluster before defining a public Operator -API. - -## Open questions - -- Which initialization steps can each Agent binary safely complete before - activation? -- What are the correct ownership groups for core Agent, trace-agent, logs, - process-agent, system-probe, and security-agent? -- Can every network probe be replaced with a reliable exec probe without - changing existing health semantics? -- How should an active process prove a UDS pathname is safe to unlink after an - abnormal old-process exit? -- Which supported clients reconnect from old stream and datagram UDS socket - objects, and on what retry schedule? -- Do any internal or external Services rely on named Agent target ports? -- Is a one-time conventional bootstrap rollout acceptable, or must the first - prepared rollout interoperate with an old Agent that does not hold locks? -- Which configuration and resource changes can OpenKruise update in place for - the production multi-container Agent Pod? -- Is UDS inode continuity valuable enough to justify a permanent endpoint - holder? -- What authenticated status channel should carry Prepared, handoff-approved, - and per-component Active acknowledgements? - -## References - -- [Kubernetes `RollingUpdateDaemonSet` API](https://github.com/kubernetes/api/blob/v0.35.3/apps/v1/types.go#L609-L646) -- [Kubernetes host-network port defaulting](https://github.com/kubernetes/kubernetes/blob/v1.35.3/pkg/apis/core/v1/defaults.go#L396-L405) -- [Kubernetes scheduler host-port filter](https://github.com/kubernetes/kubernetes/blob/v1.35.3/pkg/scheduler/framework/plugins/nodeports/node_ports.go) -- [Kubernetes DaemonSet rolling-update implementation](https://github.com/kubernetes/kubernetes/blob/v1.35.3/pkg/controller/daemon/update.go) -- [Kubernetes DaemonSet per-node Pod management](https://github.com/kubernetes/kubernetes/blob/v1.35.3/pkg/controller/daemon/daemon_controller.go) -- [Kubernetes DaemonSet surge KEP](https://github.com/kubernetes/enhancements/tree/master/keps/sig-apps/1591-daemonset-surge) -- [Kubernetes Pod resource management](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) -- [Kubernetes Pod priority and preemption](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/) -- [Kubernetes in-place Pod resize](https://kubernetes.io/docs/tasks/configure-pod-container/resize-container-resources/) -- [Kubernetes Dynamic Resource Allocation](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/) -- [Kubernetes Service traffic policy](https://kubernetes.io/docs/concepts/services-networking/service-traffic-policy/) -- [Kubernetes CSI volumes](https://kubernetes.io/docs/concepts/storage/volumes/#csi) -- [OpenKruise v1.9 Advanced DaemonSet](https://github.com/openkruise/openkruise.io/blob/55e5c2228ac27026ced2ff1ec5384966cd59e71e/versioned_docs/version-v1.9/user-manuals/advanceddaemonset.md) -- [OpenKruise in-place update semantics](https://github.com/openkruise/openkruise.io/blob/55e5c2228ac27026ced2ff1ec5384966cd59e71e/docs/core-concepts/inplace-update.md) -- [OpenKruise ImagePullJob](https://openkruise.io/docs/user-manuals/imagepulljob) -- [OpenKruise Advanced DaemonSet rollout implementation](https://github.com/openkruise/kruise/blob/07169cfac7b9cf7800dda1b8652f850cc3184132/pkg/controller/daemonset/daemonset_update.go) -- [OpenKruise image pre-download implementation](https://github.com/openkruise/kruise/blob/07169cfac7b9cf7800dda1b8652f850cc3184132/pkg/controller/daemonset/daemonset_predownload_image.go) -- [`unix(7)` Unix-domain socket semantics](https://man7.org/linux/man-pages/man7/unix.7.html) -- [Prepared host-network surge PoC](agent_host_network_surge_poc.md) diff --git a/experiments/openkruise-prepull/README.md b/experiments/openkruise-prepull/README.md deleted file mode 100644 index d03222306a..0000000000 --- a/experiments/openkruise-prepull/README.md +++ /dev/null @@ -1,378 +0,0 @@ -# OpenKruise in-place update and image pre-pull experiment - -Run on 2026-07-22 against `kind-zero-gap-agent-rollout` with Kubernetes -v1.36.1 and OpenKruise v1.9.1. The workload uses one HTTP server and a -separate observer issuing numbered requests approximately every 100 ms. An -outage is reported as the interval between the last successful request before -an update and the first successful request after it. A failed request takes up -to one second because of the observer's timeout. - -This is a behavior experiment, not a performance benchmark. The images are -small and each timing case was run once. - -## Result - -`InPlaceIfPossible` preserved the Pod for image-only changes, but it did not -overlap old and new containers. The old container was stopped before a -replacement could be pulled and started. - -| Case | Pod identity | ImagePullJob | Observed result | -|---|---|---|---| -| Nonexistent image, no gate | UID stayed `48b32d66-337b-4945-832b-834e207757d9` | None | Old container exited and the Pod remained in `ImagePullBackOff` until manual rollback | -| Uncached `python:3.13-alpine`, no gate | Same UID; container ID changed to `8ecb557e...` | None | Kubelet pull took 2.559 s; success-to-success traffic gap was 7.845 s | -| Pre-pulled `python:3.14-alpine`, explicit gate | Same UID; container ID changed to `09725b13...` | `desired=1`, `succeeded=1`, `failed=0`, `active=0` before mutation | Image was already present; success-to-success traffic gap was 4.511 s | -| Nonexistent image, explicit gate | Pod UID, container ID `09725b13...`, image, and restart count stayed unchanged | `desired=1`, `succeeded=0`, `failed=1`, `active=0` | DaemonSet was not mutated and traffic remained successful | -| Pre-pull succeeds, process fails | Same UID; container ID changed to `32266554...` | Alpine pre-pull succeeded | Existing Python command exited 127; Pod entered `CrashLoopBackOff` and traffic stayed down until rollback | -| Unsupported environment change | UID changed from `48b32d66-...` to `02b00063-...` | Not applicable | `InPlaceIfPossible` fell back to Pod recreation; traffic gap was 5.635 s | - -The single, non-comparable runs observed gaps of 7.845 s and 4.511 s, a -3.333-second difference. The cached activation reported no kubelet pull, but -the runs used different tags and do not establish the delta as a causal effect -of pre-pulling. The cached run still had a container restart and readiness gap. - -Exact primary-workload identities, in observation order: - -```text -baseline UID: 48b32d66-337b-4945-832b-834e207757d9 -baseline container: c23b14b140f52e68af62f8f35c4dd6842b988ab28928677af7d12ded7957c7a1 -rollback container: f00d1aa7ef5ae397d8b97b6abff8f6e9d68d00b6fa47b16bc48fe474738b5310 -ungated container: 8ecb557e38acd3ad2b12c49c87e4e42229c70bbe17a131a9d14fb3b0b05e8e6c -pre-pulled container: 09725b13397863d1fbfc1725ddd95e45629082ecc044c8d98aa5abcb0698084f -failed-start container: 3226655429bfb858dcbd5e1ce355fd54b9a91c6bf8bd1a06b7b876626e07999b -pre-recreation container: 5153a3b55ea7771f9a00844b66b36b866ab392799d2fdc45c7bc368e5307676c -replacement UID: 02b00063-124a-4120-a241-9e46997cdc35 -replacement container: 5c3cb507bcc83554a76cd210b25719767c67f2e6b4022463253bf590989194cd -``` - -## Evidence - -Healthy baseline: - -```text -inplace-demo-l4mj7 48b32d66-337b-4945-832b-834e207757d9 true 0 -containerd://c23b14b1... python:3.12-alpine -``` - -For the ungated missing image, the update was submitted at `15:17:21Z`. -Kubelet reported that the old container finished at `15:17:23Z`, followed by -`ImagePullBackOff`. Observer transitions were: - -```text -2026-07-22T15:17:21.134632429Z 977 OK -2026-07-22T15:17:22.236060275Z 978 FAIL -2026-07-22T15:17:50.992993345Z 1005 OK # only after rollback -``` - -The uncached valid update was submitted at `15:18:32Z`. The kubelet event was: - -```text -Successfully pulled image "python:3.13-alpine" in 2.559s -``` - -Its request boundary was: - -```text -2026-07-22T15:18:33.409066694Z 1411 OK -2026-07-22T15:18:34.532342554Z 1412 FAIL -2026-07-22T15:18:41.253995616Z 1419 OK -``` - -The standalone successful gate ran before the DaemonSet mutation: - -```text -startTime: 2026-07-22T15:19:22Z -completionTime: 2026-07-22T15:19:27Z -desired: 1 -active: 0 -succeeded: 1 -failed: 0 -``` - -After activation, kubelet reported the image was already present. Its request -boundary was: - -```text -2026-07-22T15:19:44.684584633Z 2036 OK -2026-07-22T15:19:45.786420014Z 2037 FAIL -2026-07-22T15:19:49.196018154Z 2041 OK -``` - -The standalone failed gate completed in five seconds: - -```text -startTime: 2026-07-22T15:20:27Z -completionTime: 2026-07-22T15:20:32Z -desired: 1 -active: 0 -succeeded: 0 -failed: 1 -failedNodes: [zero-gap-agent-rollout-worker] -``` - -Because the experiment did not patch the DaemonSet after that result, the Pod -remained Ready with the same UID, container ID, restart count, and image. The -next twenty observer requests were all successful. - -The activation-failure job successfully cached `alpine:3.22` at `15:21:00Z`. -After activation, the unchanged `python -m http.server` command exited 127 and -the first failed request followed the last success: - -```text -2026-07-22T15:21:19.211459847Z 2922 OK -2026-07-22T15:21:20.313133586Z 2923 FAIL -``` - -Adding an environment variable is not an eligible in-place image change. The -old Pod was deleted and the replacement became ready with a new UID. The -request boundary was: - -```text -2026-07-22T15:22:10.048879229Z 3087 OK -2026-07-22T15:22:11.150162924Z 3088 FAIL -2026-07-22T15:22:15.683683204Z 3093 OK -``` - -## Built-in AdvancedDaemonSet pre-download - -OpenKruise v1.9.1 ships `PreDownloadImageForDaemonSetUpdate` as an alpha -feature gate that defaults to false. Merely enabling `ImagePullJobGate` runs -standalone jobs but does not enable automatic AdvancedDaemonSet pre-download. - -The automatic code also skips pre-download when all Pods can update in one -batch. A separate two-worker DaemonSet with `maxUnavailable: 1` was therefore -used. After temporarily starting the managers with: - -```text ---feature-gates=ImagePullJobGate=true,PreDownloadImageForDaemonSetUpdate=true -``` - -an update to a nonexistent image produced the owned job -`automatic-predownload-demo-544cd9c95d-server` at `15:26:32Z`. One second -later the job was active and both Pods were still Ready. The controller did not -wait for it: the selected Pod's old container exited at `15:26:34Z`. At -`15:26:41Z` the job had `active=1`, `failed=1`, while that Pod was already in -`ImagePullBackOff`. The other Pod stayed healthy only because -`maxUnavailable: 1` stopped the rollout globally; the per-node availability -invariant was violated on the updated node. - -The feature flag was restored to its original value after the test: - -```text ---feature-gates=ImagePullJobGate=true -``` - -## Reproduction - -Prerequisites are the `kind-zero-gap-agent-rollout` context, two schedulable -workers named `zero-gap-agent-rollout-worker` and -`zero-gap-agent-rollout-worker2`, OpenKruise v1.9.1 with its manager and node -daemon healthy, and registry access for the fixture images. The checked-in -selectors intentionally depend on those worker names. - -```sh -# Reset only the isolated experiment namespace. -kubectl --context kind-zero-gap-agent-rollout delete namespace \ - openkruise-prepull-lab --ignore-not-found --wait=true -kubectl --context kind-zero-gap-agent-rollout apply \ - -f experiments/openkruise-prepull/base.yaml -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab wait --for=condition=Ready pod \ - -l app=inplace-demo --timeout=180s - -# Missing image without a gate; inspect ImagePullBackOff, then roll back. -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab patch daemonset.apps.kruise.io inplace-demo \ - --type=json \ - -p='[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"python:zero-gap-tag-does-not-exist"}]' -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab wait \ - --for=jsonpath='{.status.containerStatuses[0].state.waiting.reason}'=ImagePullBackOff \ - pod -l app=inplace-demo --timeout=180s -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab get pod -l app=inplace-demo -o wide -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab patch daemonset.apps.kruise.io inplace-demo \ - --type=json \ - -p='[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"python:3.12-alpine"}]' -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab wait \ - --for=jsonpath='{.status.containerStatuses[0].image}'=docker.io/library/python:3.12-alpine \ - pod -l app=inplace-demo --timeout=180s -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab wait --for=condition=Ready pod \ - -l app=inplace-demo --timeout=180s - -# Ungated image-only update -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab patch daemonset.apps.kruise.io inplace-demo \ - --type=json \ - -p='[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"python:3.13-alpine"}]' -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab wait \ - --for=jsonpath='{.status.containerStatuses[0].image}'=docker.io/library/python:3.13-alpine \ - pod -l app=inplace-demo --timeout=180s -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab wait --for=condition=Ready pod \ - -l app=inplace-demo --timeout=180s - -# Explicit pre-pull gate -kubectl --context kind-zero-gap-agent-rollout apply \ - -f experiments/openkruise-prepull/imagepulljob-good.yaml -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab get imagepulljob python-3-14-alpine -o yaml -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab wait \ - --for=jsonpath='{.status.completionTime}' imagepulljob/python-3-14-alpine \ - --timeout=360s -test "$(kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab get imagepulljob python-3-14-alpine \ - -o jsonpath='{.status.desired},{.status.succeeded},{.status.failed},{.status.active}')" \ - = "1,1,0,0" - -# Mutate the DaemonSet only after desired > 0, succeeded == desired, -# failed == 0, active == 0, and completionTime is set. -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab patch daemonset.apps.kruise.io inplace-demo \ - --type=json \ - -p='[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"python:3.14-alpine"}]' -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab wait \ - --for=jsonpath='{.status.containerStatuses[0].image}'=docker.io/library/python:3.14-alpine \ - pod -l app=inplace-demo --timeout=180s -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab wait --for=condition=Ready pod \ - -l app=inplace-demo --timeout=180s - -# Failed gate: inspect the result and deliberately do not patch the DaemonSet. -kubectl --context kind-zero-gap-agent-rollout apply \ - -f experiments/openkruise-prepull/imagepulljob-bad.yaml -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab get imagepulljob python-missing -o yaml -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab wait \ - --for=jsonpath='{.status.completionTime}' imagepulljob/python-missing \ - --timeout=120s -test "$(kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab get imagepulljob python-missing \ - -o jsonpath='{.status.desired},{.status.succeeded},{.status.failed},{.status.active}')" \ - = "1,0,1,0" - -# Successfully cached image with an invalid retained command. -kubectl --context kind-zero-gap-agent-rollout apply \ - -f experiments/openkruise-prepull/imagepulljob-activation-failure.yaml -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab wait \ - --for=jsonpath='{.status.completionTime}' imagepulljob/alpine-3-22 \ - --timeout=360s -test "$(kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab get imagepulljob alpine-3-22 \ - -o jsonpath='{.status.desired},{.status.succeeded},{.status.failed},{.status.active}')" \ - = "1,1,0,0" -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab patch daemonset.apps.kruise.io inplace-demo \ - --type=json \ - -p='[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"alpine:3.22"}]' -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab wait \ - --for=jsonpath='{.status.containerStatuses[0].state.waiting.reason}'=CrashLoopBackOff \ - pod -l app=inplace-demo --timeout=180s -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab patch daemonset.apps.kruise.io inplace-demo \ - --type=json \ - -p='[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"python:3.14-alpine"}]' -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab wait \ - --for=jsonpath='{.status.containerStatuses[0].image}'=docker.io/library/python:3.14-alpine \ - pod -l app=inplace-demo --timeout=180s -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab wait --for=condition=Ready pod \ - -l app=inplace-demo --timeout=180s - -# Unsupported Pod-template change. Record and require a new Pod UID. -inplace_uid=$(kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab get pod -l app=inplace-demo \ - -o jsonpath='{.items[0].metadata.uid}') -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab patch daemonset.apps.kruise.io inplace-demo \ - --type=json \ - -p='[{"op":"add","path":"/spec/template/spec/containers/0/env","value":[{"name":"UNSUPPORTED_TEMPLATE_CHANGE","value":"true"}]}]' -while test "$(kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab get pod -l app=inplace-demo \ - -o jsonpath='{.items[0].metadata.uid}' 2>/dev/null)" = "$inplace_uid"; do sleep 1; done -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab wait --for=condition=Ready pod \ - -l app=inplace-demo --timeout=180s - -# Two-worker automatic pre-download fixture. -kubectl --context kind-zero-gap-agent-rollout apply \ - -f experiments/openkruise-prepull/automatic-predownload.yaml -# Do not continue until both worker Pods are Ready. -while test "$(kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab get daemonset.apps.kruise.io \ - automatic-predownload-demo -o jsonpath='{.status.numberReady}')" != "2"; do sleep 1; done -# Verify the current argument before using the observed index from this install. -kubectl --context kind-zero-gap-agent-rollout -n kruise-system get deployment \ - kruise-controller-manager \ - -o jsonpath='{.spec.template.spec.containers[0].args[6]}{"\n"}' -# Abort unless the preceding command prints exactly: -# --feature-gates=ImagePullJobGate=true -kubectl --context kind-zero-gap-agent-rollout -n kruise-system patch deployment \ - kruise-controller-manager --type=json \ - -p='[{"op":"replace","path":"/spec/template/spec/containers/0/args/6","value":"--feature-gates=ImagePullJobGate=true,PreDownloadImageForDaemonSetUpdate=true"}]' -kubectl --context kind-zero-gap-agent-rollout -n kruise-system rollout status \ - deployment/kruise-controller-manager --timeout=180s -kubectl --context kind-zero-gap-agent-rollout -n kruise-system get deployment \ - kruise-controller-manager \ - -o jsonpath='{.spec.template.spec.containers[0].args[6]}{"\n"}' -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab patch daemonset.apps.kruise.io \ - automatic-predownload-demo --type=json \ - -p='[{"op":"replace","path":"/spec/template/spec/containers/0/image","value":"python:zero-gap-auto-gate-does-not-exist"}]' -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab get imagepulljobs.apps.kruise.io -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab get pods -l app=automatic-predownload-demo -o wide - -# Observer transition extraction used for each timing boundary. -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab logs observer --timestamps \ - --since-time=2026-07-22T15:19:43Z \ - | awk '/ (OK|FAIL)$/ {state=$NF; if (state != previous) {print; previous=state}}' - -# Restore the two-worker fixture and original manager feature flags. -kubectl --context kind-zero-gap-agent-rollout apply \ - -f experiments/openkruise-prepull/automatic-predownload.yaml -while test "$(kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab get daemonset.apps.kruise.io \ - automatic-predownload-demo -o jsonpath='{.status.numberReady}')" != "2"; do sleep 1; done -kubectl --context kind-zero-gap-agent-rollout -n kruise-system patch deployment \ - kruise-controller-manager --type=json \ - -p='[{"op":"replace","path":"/spec/template/spec/containers/0/args/6","value":"--feature-gates=ImagePullJobGate=true"}]' -kubectl --context kind-zero-gap-agent-rollout -n kruise-system rollout status \ - deployment/kruise-controller-manager --timeout=180s -kubectl --context kind-zero-gap-agent-rollout apply \ - -f experiments/openkruise-prepull/base.yaml -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab wait \ - --for=jsonpath='{.status.containerStatuses[0].image}'=docker.io/library/python:3.12-alpine \ - pod -l app=inplace-demo --timeout=180s -kubectl --context kind-zero-gap-agent-rollout \ - -n openkruise-prepull-lab get pods -o wide -kubectl --context kind-zero-gap-agent-rollout -n kruise-system get deployment \ - kruise-controller-manager \ - -o jsonpath='{.spec.template.spec.containers[0].args[6]}{"\n"}' -``` - -For a production gate, use an immutable digest with `IfNotPresent`. A mutable -tag or `Always` can still require registry resolution during activation. Image -garbage collection or a node restart between gate completion and activation -can also evict the cached digest, so this is not a permanent pull guarantee. - -## Conclusion - -Image pre-pulling is worthwhile preparation. The built-in automatic mechanism -is not an availability gate, while a standalone Operator-managed ImagePullJob -can gate DaemonSet mutation on a successful pull while that cached image -remains available. Neither form validates process startup, and in-place update -still has a restart gap. It is therefore a complementary optimization, not the -zero-gap primitive. diff --git a/experiments/openkruise-prepull/automatic-predownload.yaml b/experiments/openkruise-prepull/automatic-predownload.yaml deleted file mode 100644 index 4f14568abd..0000000000 --- a/experiments/openkruise-prepull/automatic-predownload.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: apps.kruise.io/v1beta1 -kind: DaemonSet -metadata: - name: automatic-predownload-demo - namespace: openkruise-prepull-lab -spec: - selector: - matchLabels: - app: automatic-predownload-demo - updateStrategy: - type: RollingUpdate - rollingUpdate: - rollingUpdateType: InPlaceIfPossible - maxUnavailable: 1 - template: - metadata: - labels: - app: automatic-predownload-demo - spec: - terminationGracePeriodSeconds: 1 - containers: - - name: server - image: python:3.12-alpine - imagePullPolicy: IfNotPresent - command: - - /bin/sh - - -c - - | - mkdir -p /www - printf 'ok\n' >/www/index.html - exec python -m http.server 8080 --directory /www - readinessProbe: - httpGet: - path: / - port: 8080 - periodSeconds: 1 - timeoutSeconds: 1 - resources: - requests: - cpu: 10m - memory: 8Mi - limits: - memory: 32Mi diff --git a/experiments/openkruise-prepull/base.yaml b/experiments/openkruise-prepull/base.yaml deleted file mode 100644 index fde53ee5db..0000000000 --- a/experiments/openkruise-prepull/base.yaml +++ /dev/null @@ -1,100 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: openkruise-prepull-lab ---- -apiVersion: apps.kruise.io/v1beta1 -kind: DaemonSet -metadata: - name: inplace-demo - namespace: openkruise-prepull-lab -spec: - selector: - matchLabels: - app: inplace-demo - updateStrategy: - type: RollingUpdate - rollingUpdate: - rollingUpdateType: InPlaceIfPossible - maxUnavailable: 1 - template: - metadata: - labels: - app: inplace-demo - spec: - nodeSelector: - kubernetes.io/hostname: zero-gap-agent-rollout-worker - terminationGracePeriodSeconds: 1 - containers: - - name: server - image: python:3.12-alpine - imagePullPolicy: IfNotPresent - command: - - /bin/sh - - -c - - | - mkdir -p /www - printf 'ok\n' >/www/index.html - exec python -m http.server 8080 --directory /www - ports: - - name: http - containerPort: 8080 - readinessProbe: - httpGet: - path: / - port: 8080 - periodSeconds: 1 - timeoutSeconds: 1 - resources: - requests: - cpu: 10m - memory: 8Mi - limits: - memory: 32Mi ---- -apiVersion: v1 -kind: Service -metadata: - name: inplace-demo - namespace: openkruise-prepull-lab -spec: - selector: - app: inplace-demo - ports: - - name: http - port: 80 - targetPort: http ---- -apiVersion: v1 -kind: Pod -metadata: - name: observer - namespace: openkruise-prepull-lab -spec: - nodeSelector: - kubernetes.io/hostname: zero-gap-agent-rollout-worker2 - restartPolicy: Never - containers: - - name: observer - image: alpine:3.20 - imagePullPolicy: IfNotPresent - command: - - /bin/sh - - -c - - | - sequence=0 - while true; do - sequence=$((sequence + 1)) - if wget -q -T 1 -O /dev/null http://inplace-demo; then - printf '%d OK\n' "${sequence}" - else - printf '%d FAIL\n' "${sequence}" - fi - sleep 0.1 - done - resources: - requests: - cpu: 5m - memory: 4Mi - limits: - memory: 16Mi diff --git a/experiments/openkruise-prepull/imagepulljob-activation-failure.yaml b/experiments/openkruise-prepull/imagepulljob-activation-failure.yaml deleted file mode 100644 index 1f6cc89437..0000000000 --- a/experiments/openkruise-prepull/imagepulljob-activation-failure.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: apps.kruise.io/v1alpha1 -kind: ImagePullJob -metadata: - name: alpine-3-22 - namespace: openkruise-prepull-lab -spec: - image: alpine:3.22 - imagePullPolicy: IfNotPresent - selector: - matchLabels: - kubernetes.io/hostname: zero-gap-agent-rollout-worker - parallelism: 1 - pullPolicy: - timeoutSeconds: 300 - backoffLimit: 1 - completionPolicy: - type: Always - activeDeadlineSeconds: 360 diff --git a/experiments/openkruise-prepull/imagepulljob-bad.yaml b/experiments/openkruise-prepull/imagepulljob-bad.yaml deleted file mode 100644 index 793b6cd5ef..0000000000 --- a/experiments/openkruise-prepull/imagepulljob-bad.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: apps.kruise.io/v1alpha1 -kind: ImagePullJob -metadata: - name: python-missing - namespace: openkruise-prepull-lab -spec: - image: python:zero-gap-tag-does-not-exist - imagePullPolicy: Always - selector: - matchLabels: - kubernetes.io/hostname: zero-gap-agent-rollout-worker - parallelism: 1 - pullPolicy: - timeoutSeconds: 60 - backoffLimit: 0 - completionPolicy: - type: Always - activeDeadlineSeconds: 90 diff --git a/experiments/openkruise-prepull/imagepulljob-good.yaml b/experiments/openkruise-prepull/imagepulljob-good.yaml deleted file mode 100644 index bfdb8ca6c4..0000000000 --- a/experiments/openkruise-prepull/imagepulljob-good.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: apps.kruise.io/v1alpha1 -kind: ImagePullJob -metadata: - name: python-3-14-alpine - namespace: openkruise-prepull-lab -spec: - image: python:3.14-alpine - imagePullPolicy: IfNotPresent - selector: - matchLabels: - kubernetes.io/hostname: zero-gap-agent-rollout-worker - parallelism: 1 - pullPolicy: - timeoutSeconds: 300 - backoffLimit: 1 - completionPolicy: - type: Always - activeDeadlineSeconds: 360 diff --git a/internal/controller/datadogagentinternal/controller_reconcile_agent.go b/internal/controller/datadogagentinternal/controller_reconcile_agent.go index 36147f414d..e9c148bfae 100644 --- a/internal/controller/datadogagentinternal/controller_reconcile_agent.go +++ b/internal/controller/datadogagentinternal/controller_reconcile_agent.go @@ -16,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" @@ -125,8 +126,8 @@ func (r *Reconciler) reconcileV2Agent(ctx context.Context, requiredComponents fe true, ) } - if deleteErr := r.deleteV2ExtendedDaemonSet(ctx, ddai, eds, newStatus); deleteErr != nil { - return reconcile.Result{}, deleteErr + if err := r.deleteV2ExtendedDaemonSet(ctx, ddai, eds, newStatus); err != nil { + return reconcile.Result{}, err } return reconcile.Result{}, nil } @@ -219,8 +220,8 @@ func (r *Reconciler) reconcileV2Agent(ctx context.Context, requiredComponents fe daemonsetLogger.Info("Removing Windows DaemonSet: FIPS (useFIPSAgent or fips.enabled) is enabled and FIPS is unsupported on Windows") // Delete any existing Windows DaemonSet so a non-FIPS agent does not keep running // under a FIPS-required configuration (compliance), rather than silently downgrading. - if deleteErr := r.deleteV2DaemonSet(ctx, ddai, daemonset, newStatus); deleteErr != nil { - return reconcile.Result{}, deleteErr + if err := r.deleteV2DaemonSet(ctx, ddai, daemonset, newStatus); err != nil { + return reconcile.Result{}, err } // Clear the Agent status unconditionally: deleteV2DaemonSet returns early (without // clearing status) if the DaemonSet is already gone, which would otherwise leave a @@ -283,15 +284,30 @@ func (r *Reconciler) reconcileV2Agent(ctx context.Context, requiredComponents fe true, ) } - if deleteErr := r.deleteV2DaemonSet(ctx, ddai, daemonset, newStatus); deleteErr != nil { - return reconcile.Result{}, deleteErr + if err := r.deleteV2DaemonSet(ctx, ddai, daemonset, newStatus); err != nil { + return reconcile.Result{}, err } deleteStatusWithAgent(newStatus) return reconcile.Result{}, nil } - rolloutBudget := resourceFallbackBudget(ddai, &r.options.ExtendedDaemonsetOptions) - preparedPhase, prepareErr := r.configurePreparedRollout(ctx, ddai, daemonset, rolloutBudget) + rolloutBudget := preparedRolloutBudget(ddai, &r.options.ExtendedDaemonsetOptions) + rolloutEnabled := preparedRolloutEnabled(ddai) + var currentDaemonSet *appsv1.DaemonSet + if rolloutEnabled { + reader := r.apiReader + if reader == nil { + reader = r.client + } + currentDaemonSet = &appsv1.DaemonSet{} + if getErr := reader.Get(ctx, client.ObjectKeyFromObject(daemonset), currentDaemonSet); getErr != nil { + if !errors.IsNotFound(getErr) { + return reconcile.Result{}, getErr + } + currentDaemonSet = nil + } + } + affinityMigration, prepareErr := configurePreparedRollout(ddai, daemonset, currentDaemonSet, rolloutBudget) if prepareErr != nil { objLogger.Error(prepareErr, "Prepared Agent rollout request is incompatible with the rendered Pod template") if r.recorder != nil { @@ -300,27 +316,21 @@ func (r *Reconciler) reconcileV2Agent(ctx context.Context, requiredComponents fe return reconcile.Result{}, prepareErr } result, err := r.createOrUpdateDaemonset(ctx, ddai, daemonset, newStatus, updateDSStatusV2WithAgent) - if err != nil || preparedPhase == "" { + if err != nil || !rolloutEnabled { return result, err } - requeuePreparedArm(&result, preparedPhase) - if preparedPhase == preparedRolloutPhaseStandby { - handoffResult, handoffErr := r.reconcilePreparedHandoff(ctx, ddai, daemonset, rolloutBudget) - if handoffErr != nil { - return reconcile.Result{}, handoffErr - } - if handoffResult.RequeueAfter > 0 && (result.RequeueAfter == 0 || handoffResult.RequeueAfter < result.RequeueAfter) { - result.RequeueAfter = handoffResult.RequeueAfter + if affinityMigration { + if result.RequeueAfter == 0 || result.RequeueAfter > time.Second { + result.RequeueAfter = time.Second } + return result, nil } - if resourceFallbackEnabled(ddai) { - fallbackResult, fallbackErr := r.reconcileResourceFallback(ctx, ddai, daemonset, rolloutBudget) - if fallbackErr != nil { - return reconcile.Result{}, fallbackErr - } - if fallbackResult.RequeueAfter > 0 && (result.RequeueAfter == 0 || fallbackResult.RequeueAfter < result.RequeueAfter) { - result.RequeueAfter = fallbackResult.RequeueAfter - } + fallbackResult, fallbackErr := r.reconcileResourceFallback(ctx, ddai, daemonset, rolloutBudget) + if fallbackErr != nil { + return reconcile.Result{}, fallbackErr + } + if fallbackResult.RequeueAfter > 0 && (result.RequeueAfter == 0 || fallbackResult.RequeueAfter < result.RequeueAfter) { + result.RequeueAfter = fallbackResult.RequeueAfter } return result, nil } diff --git a/internal/controller/datadogagentinternal/controller_reconcile_agent_test.go b/internal/controller/datadogagentinternal/controller_reconcile_agent_test.go index 2a7c815553..fa95f12eb0 100644 --- a/internal/controller/datadogagentinternal/controller_reconcile_agent_test.go +++ b/internal/controller/datadogagentinternal/controller_reconcile_agent_test.go @@ -3,7 +3,6 @@ package datadogagentinternal import ( "context" "testing" - "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -11,7 +10,6 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/tools/record" "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" apicommon "github.com/DataDog/datadog-operator/api/datadoghq/common" @@ -33,29 +31,8 @@ import ( const defaultProvider = kubernetes.DefaultProvider const gkeCosProvider = kubernetes.GKECloudProvider + "-" + kubernetes.GKECosType -func TestReconcileV2AgentRejectsInvalidPreparedRolloutBeforeCreatingDaemonSet(t *testing.T) { +func TestReconcileV2AgentCreatesPreparedSurgeDaemonSet(t *testing.T) { r, ddai := newPreparedRolloutReconciler(t, false) - - result, err := r.reconcileV2Agent( - context.Background(), - preparedRolloutRequiredComponents(), - nil, - ddai, - feature.NewResourceManagers(store.NewStore(ddai, nil)), - &datadoghqv1alpha1.DatadogAgentInternalStatus{}, - defaultProvider, - ) - - require.ErrorContains(t, err, "hostNetwork=true") - assert.Zero(t, result.RequeueAfter) - daemonSets := &appsv1.DaemonSetList{} - require.NoError(t, r.client.List(context.Background(), daemonSets)) - assert.Empty(t, daemonSets.Items, "an incompatible prepared rollout must fail before creating a DaemonSet") -} - -func TestReconcileV2AgentCreatesArmedPreparedDaemonSet(t *testing.T) { - r, ddai := newPreparedRolloutReconciler(t, true) - ddai.Annotations[resourceFallbackAnnotation] = "true" status := &datadoghqv1alpha1.DatadogAgentInternalStatus{} result, err := r.reconcileV2Agent( @@ -69,38 +46,13 @@ func TestReconcileV2AgentCreatesArmedPreparedDaemonSet(t *testing.T) { ) require.NoError(t, err) - assert.Equal(t, time.Second, result.RequeueAfter) + assert.Zero(t, result.RequeueAfter) daemonSets := &appsv1.DaemonSetList{} require.NoError(t, r.client.List(context.Background(), daemonSets)) require.Len(t, daemonSets.Items, 1) ds := &daemonSets.Items[0] - assert.Equal(t, preparedRolloutPhaseArm, ds.Spec.Template.Annotations[preparedRolloutPhaseAnnotation]) + assert.Equal(t, preparedRolloutModeV1, ds.Spec.Template.Annotations[preparedRolloutModeAnnotation]) require.NotNil(t, ds.Spec.UpdateStrategy.RollingUpdate) - assert.Equal(t, intstr.FromInt(0), *ds.Spec.UpdateStrategy.RollingUpdate.MaxSurge) - assert.Equal(t, intstr.FromInt(1), *ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable) - - ds.Status = appsv1.DaemonSetStatus{ - ObservedGeneration: ds.Generation, - DesiredNumberScheduled: 1, - UpdatedNumberScheduled: 1, - NumberReady: 1, - NumberAvailable: 1, - } - require.NoError(t, r.client.Status().Update(context.Background(), ds)) - - result, err = r.reconcileV2Agent( - context.Background(), - preparedRolloutRequiredComponents(), - nil, - ddai, - feature.NewResourceManagers(store.NewStore(ddai, nil)), - status, - defaultProvider, - ) - require.NoError(t, err) - assert.Zero(t, result.RequeueAfter, "standby without a handoff candidate does not need an extra poll") - require.NoError(t, r.client.Get(context.Background(), client.ObjectKeyFromObject(ds), ds)) - assert.Equal(t, preparedRolloutPhaseStandby, ds.Spec.Template.Annotations[preparedRolloutPhaseAnnotation]) assert.Equal(t, intstr.FromInt(1), *ds.Spec.UpdateStrategy.RollingUpdate.MaxSurge) assert.Equal(t, intstr.FromInt(0), *ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable) } @@ -115,7 +67,7 @@ func newPreparedRolloutReconciler(t *testing.T, hostNetwork bool) (*Reconciler, one := intstr.FromInt(1) ddai := pkgtestutils.NewDatadogAgentInternal("datadog-agent", "agent", nil) ddai.UID = "ddai-uid" - ddai.Annotations = map[string]string{preparedRolloutAnnotation: "true"} + ddai.Annotations = map[string]string{preparedRolloutModeAnnotation: preparedRolloutModeV1} ddai.Spec.Features = &datadoghqv2alpha1.DatadogFeatures{} ddai.Spec.Override = map[datadoghqv2alpha1.ComponentName]*datadoghqv2alpha1.DatadogAgentComponentOverride{ datadoghqv2alpha1.NodeAgentComponentName: { diff --git a/internal/controller/datadogagentinternal/prepared_rollout.go b/internal/controller/datadogagentinternal/prepared_rollout.go index f675b5423e..adea3a9ef9 100644 --- a/internal/controller/datadogagentinternal/prepared_rollout.go +++ b/internal/controller/datadogagentinternal/prepared_rollout.go @@ -5,43 +5,29 @@ package datadogagentinternal import ( - "context" - "crypto/sha256" - "encoding/json" "fmt" - "sort" + "path" "strings" - "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/util/intstr" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/reconcile" apicommon "github.com/DataDog/datadog-operator/api/datadoghq/common" datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" ) const ( - preparedRolloutAnnotation = "experimental.agent.datadoghq.com/host-network-surge-prepared" - preparedRolloutPhaseAnnotation = "experimental.agent.datadoghq.com/prepared-rollout-phase" - preparedRolloutArmHashAnnotation = "experimental.agent.datadoghq.com/prepared-rollout-arm-hash" - resourceFallbackAnnotation = "experimental.agent.datadoghq.com/resource-fallback" - preparedRolloutPhaseArm = "arm" - preparedRolloutPhaseStandby = "standby" + preparedRolloutModeAnnotation = "experimental.agent.datadoghq.com/node-agent-rollout-mode" + preparedRolloutModeV1 = "prepared-surge-v1" - preparedRolloutLockVolume = "agent-rollout-locks" preparedRolloutStateVolume = "agent-rollout-state" - preparedRolloutLockDir = "/var/run/datadog-agent-rollout" preparedRolloutStateDir = "/var/run/datadog-agent-rollout-state" - preparedRolloutRequeue = time.Second rolloutEnabledEnv = "DD_EXPERIMENTAL_NODE_AGENT_ROLLOUT_ENABLED" - rolloutLockPathEnv = "DD_EXPERIMENTAL_NODE_AGENT_ROLLOUT_LOCK_PATH" rolloutStatePathEnv = "DD_EXPERIMENTAL_NODE_AGENT_ROLLOUT_STATE_PATH" + rolloutPodUIDEnv = "DD_EXPERIMENTAL_NODE_AGENT_ROLLOUT_POD_UID" ) var preparedRolloutContainerNames = []string{ @@ -50,127 +36,69 @@ var preparedRolloutContainerNames = []string{ } func preparedRolloutEnabled(ddai *datadoghqv1alpha1.DatadogAgentInternal) bool { - return strings.EqualFold(ddai.Annotations[preparedRolloutAnnotation], "true") + return ddai != nil && ddai.Annotations[preparedRolloutModeAnnotation] == preparedRolloutModeV1 } -func resourceFallbackEnabled(ddai *datadoghqv1alpha1.DatadogAgentInternal) bool { - return preparedRolloutEnabled(ddai) && strings.EqualFold(ddai.Annotations[resourceFallbackAnnotation], "true") -} - -// configurePreparedRollout installs a restart-safe two-phase protocol. A -// conventional rollout first arms every old process with the node-local lock. -// Only an exact, fully available arm revision may transition to standby surge. -func (r *Reconciler) configurePreparedRollout(ctx context.Context, ddai *datadoghqv1alpha1.DatadogAgentInternal, desired *appsv1.DaemonSet, budget intstr.IntOrString) (string, error) { +// configurePreparedRollout enables native DaemonSet surge. Profile-managed +// DaemonSets first need one conventional affinity-only rollout because an old +// Pod's broad required anti-affinity also rejects an incoming replacement. +// The returned boolean is true while that prerequisite rollout is in progress. +func configurePreparedRollout(ddai *datadoghqv1alpha1.DatadogAgentInternal, ds, current *appsv1.DaemonSet, budget intstr.IntOrString) (bool, error) { if !preparedRolloutEnabled(ddai) { - return "", nil + return false, nil } if !positiveIntOrPercent(&budget) { - return "", fmt.Errorf("prepared Agent rollout requires a positive, valid maxUnavailable budget") - } - if desired.Spec.UpdateStrategy.Type != "" && desired.Spec.UpdateStrategy.Type != appsv1.RollingUpdateDaemonSetStrategyType { - return "", fmt.Errorf("prepared Agent rollout requires RollingUpdate strategy") + return false, fmt.Errorf("prepared Agent rollout requires a positive, valid maxUnavailable budget") } - - armed := desired.DeepCopy() - if err := prepareAgentTemplate(armed, preparedRolloutPhaseArm); err != nil { - return "", err + if ds.Spec.UpdateStrategy.Type != "" && ds.Spec.UpdateStrategy.Type != appsv1.RollingUpdateDaemonSetStrategyType { + return false, fmt.Errorf("prepared Agent rollout requires RollingUpdate strategy") } - armHash, err := stampPreparedArmHash(&armed.Spec.Template) - if err != nil { - return "", err + prepared := ds.DeepCopy() + if err := prepareAgentTemplate(prepared); err != nil { + return false, err } - configureArmStrategy(armed, budget) - - live := &appsv1.DaemonSet{} - err = r.apiReader.Get(ctx, client.ObjectKeyFromObject(desired), live) - if err != nil && !apierrors.IsNotFound(err) { - return "", fmt.Errorf("get live Agent DaemonSet for prepared rollout: %w", err) + if !configurePreparedSurge(prepared, budget) { + return false, fmt.Errorf("prepared Agent rollout requires a positive, valid maxUnavailable budget") } - phase := preparedRolloutPhaseArm - if err == nil { - livePhase := live.Spec.Template.Annotations[preparedRolloutPhaseAnnotation] - switch livePhase { - case preparedRolloutPhaseStandby: - // Never oscillate back to arm during a mixed or failed surged rollout. - phase = preparedRolloutPhaseStandby - case preparedRolloutPhaseArm: - if live.Spec.Template.Annotations[preparedRolloutArmHashAnnotation] == armHash && daemonSetArmComplete(live) { - phase = preparedRolloutPhaseStandby + if current != nil && profileAffinityMigrationPending(current) { + migrationTemplate := current.Spec.Template.DeepCopy() + if apiequality.Semantic.DeepEqual(migrationTemplate.Spec.Affinity.PodAntiAffinity, broadAgentPodAntiAffinity()) { + if !prepareProfileAntiAffinityForSurge(migrationTemplate) { + return false, fmt.Errorf("prepared Agent rollout cannot migrate profile anti-affinity") } } + ds.Spec.Template = *migrationTemplate + configureConventionalMigration(ds, budget) + return true, nil } - if phase == preparedRolloutPhaseArm { - *desired = *armed - return phase, nil - } - - standby := desired.DeepCopy() - if err := prepareAgentTemplate(standby, preparedRolloutPhaseStandby); err != nil { - return "", err - } - standby.Spec.Template.Annotations[preparedRolloutArmHashAnnotation] = armHash - if !configureResourceFallback(standby, budget) { - return "", fmt.Errorf("prepared Agent rollout requires a positive, valid maxUnavailable budget") - } - *desired = *standby - return phase, nil -} - -// requeuePreparedArm keeps polling until the DaemonSet controller reports the -// fully available arm revision. DaemonSet status-only updates are filtered by -// the controller watch, so Pod and generation events alone cannot guarantee a -// final reconcile after status catches up. -func requeuePreparedArm(result *reconcile.Result, phase string) { - if phase == preparedRolloutPhaseArm && (result.RequeueAfter == 0 || preparedRolloutRequeue < result.RequeueAfter) { - result.RequeueAfter = preparedRolloutRequeue - } + ds.Spec.Template = prepared.Spec.Template + ds.Spec.UpdateStrategy = prepared.Spec.UpdateStrategy + return false, nil } -func stampPreparedArmHash(template *corev1.PodTemplateSpec) (string, error) { - templateCopy := template.DeepCopy() - delete(templateCopy.Annotations, preparedRolloutArmHashAnnotation) - serialized, err := json.Marshal(templateCopy) - if err != nil { - return "", fmt.Errorf("hash prepared Agent arm template: %w", err) +func profileAffinityMigrationPending(current *appsv1.DaemonSet) bool { + antiAffinity := current.Spec.Template.Spec.Affinity + if antiAffinity == nil || antiAffinity.PodAntiAffinity == nil { + return false } - hash := fmt.Sprintf("%x", sha256.Sum256(serialized)) - if template.Annotations == nil { - template.Annotations = map[string]string{} + if apiequality.Semantic.DeepEqual(antiAffinity.PodAntiAffinity, broadAgentPodAntiAffinity()) { + return true } - template.Annotations[preparedRolloutArmHashAnnotation] = hash - return hash, nil + expected, ok := profileSurgePodAntiAffinity(current.Spec.Template.Labels) + return ok && apiequality.Semantic.DeepEqual(antiAffinity.PodAntiAffinity, expected) && + !hasRolloutMode(current.Spec.Template.Annotations) && !daemonSetFullyRolledOut(current) } -func daemonSetArmComplete(ds *appsv1.DaemonSet) bool { +func daemonSetFullyRolledOut(ds *appsv1.DaemonSet) bool { desired := ds.Status.DesiredNumberScheduled - return desired > 0 && - ds.Status.ObservedGeneration == ds.Generation && - ds.Status.UpdatedNumberScheduled == desired && - ds.Status.NumberReady == desired && - ds.Status.NumberAvailable == desired && - ds.Status.NumberUnavailable == 0 -} - -func configureArmStrategy(ds *appsv1.DaemonSet, budget intstr.IntOrString) { - ds.Spec.UpdateStrategy.Type = appsv1.RollingUpdateDaemonSetStrategyType - if ds.Spec.UpdateStrategy.RollingUpdate == nil { - ds.Spec.UpdateStrategy.RollingUpdate = &appsv1.RollingUpdateDaemonSet{} - } - zero := intstr.FromInt(0) - ds.Spec.UpdateStrategy.RollingUpdate.MaxSurge = &zero - if positiveIntOrPercent(&budget) { - value := budget - ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable = &value - } + return desired > 0 && ds.Status.ObservedGeneration == ds.Generation && + ds.Status.UpdatedNumberScheduled == desired && ds.Status.NumberAvailable == desired && ds.Status.NumberUnavailable == 0 } -func prepareAgentTemplate(ds *appsv1.DaemonSet, phase string) error { +func prepareAgentTemplate(ds *appsv1.DaemonSet) error { spec := &ds.Spec.Template.Spec - if !spec.HostNetwork { - return fmt.Errorf("prepared Agent rollout requires hostNetwork=true") - } if spec.OS != nil && spec.OS.Name != corev1.Linux { return fmt.Errorf("prepared Agent rollout is Linux-only") } @@ -183,10 +111,12 @@ func prepareAgentTemplate(ds *appsv1.DaemonSet, phase string) error { if !prepareProfileAntiAffinityForSurge(&ds.Spec.Template) { return fmt.Errorf("prepared Agent rollout does not support custom Pod anti-affinity") } - if err := addPreparedRolloutVolumes(spec); err != nil { + if !spec.HostNetwork && podUsesHostPorts(spec) { + return fmt.Errorf("prepared Agent rollout cannot overlap Pod-networked containers that declare hostPort") + } + if err := addPreparedRolloutStateVolume(spec); err != nil { return err } - for i := range spec.Containers { container := &spec.Containers[i] if container.Name == string(apicommon.TraceAgentContainerName) { @@ -203,11 +133,14 @@ func prepareAgentTemplate(ds *appsv1.DaemonSet, phase string) error { container.Command = append([]string(nil), container.Command[traceIndex:]...) } configurePreparedContainer(container) - if phase == preparedRolloutPhaseStandby { + // With host networking, Kubernetes' scheduler treats declared container + // ports as node-local claims. The process can still bind the same host + // address after the older Pod exits without these declarations. + if spec.HostNetwork { container.Ports = nil } } - if phase == preparedRolloutPhaseStandby { + if spec.HostNetwork { for i := range spec.InitContainers { spec.InitContainers[i].Ports = nil } @@ -215,10 +148,28 @@ func prepareAgentTemplate(ds *appsv1.DaemonSet, phase string) error { if ds.Spec.Template.Annotations == nil { ds.Spec.Template.Annotations = map[string]string{} } - ds.Spec.Template.Annotations[preparedRolloutPhaseAnnotation] = phase + ds.Spec.Template.Annotations[preparedRolloutModeAnnotation] = preparedRolloutModeV1 return nil } +func podUsesHostPorts(spec *corev1.PodSpec) bool { + for i := range spec.InitContainers { + for _, port := range spec.InitContainers[i].Ports { + if port.HostPort != 0 { + return true + } + } + } + for i := range spec.Containers { + for _, port := range spec.Containers[i].Ports { + if port.HostPort != 0 { + return true + } + } + } + return false +} + func validatePreparedContainers(spec *corev1.PodSpec) error { if len(spec.Containers) != len(preparedRolloutContainerNames) { return fmt.Errorf("prepared Agent rollout initially supports exactly agent and trace-agent containers") @@ -243,7 +194,7 @@ func validatePreparedContainers(spec *corev1.PodSpec) error { return fmt.Errorf("prepared Agent rollout requires the standard agent run command") } for _, mount := range container.VolumeMounts { - if mount.Name == preparedRolloutLockVolume || mount.Name == preparedRolloutStateVolume || mount.MountPath == preparedRolloutLockDir || mount.MountPath == preparedRolloutStateDir { + if mount.Name == preparedRolloutStateVolume || mountContainsPath(mount.MountPath, preparedRolloutStateDir) { return fmt.Errorf("prepared Agent rollout volume mount on container %q conflicts with a reserved name or path", container.Name) } } @@ -251,6 +202,7 @@ func validatePreparedContainers(spec *corev1.PodSpec) error { if !seen[string(apicommon.CoreAgentContainerName)] || !seen[string(apicommon.TraceAgentContainerName)] { return fmt.Errorf("prepared Agent rollout requires agent and trace-agent containers") } + if len(spec.InitContainers) != 2 { return fmt.Errorf("prepared Agent rollout initially supports only init-volume and init-config init containers") } @@ -271,61 +223,59 @@ func validatePreparedContainers(spec *corev1.PodSpec) error { return nil } -func addPreparedRolloutVolumes(spec *corev1.PodSpec) error { +func mountContainsPath(mountPath, target string) bool { + mountPath = path.Clean(mountPath) + target = path.Clean(target) + return mountPath == "/" || target == mountPath || strings.HasPrefix(target, mountPath+"/") +} + +func addPreparedRolloutStateVolume(spec *corev1.PodSpec) error { for i := range spec.Volumes { - if spec.Volumes[i].Name == preparedRolloutLockVolume || spec.Volumes[i].Name == preparedRolloutStateVolume { - return fmt.Errorf("prepared Agent rollout volume name %q is reserved", spec.Volumes[i].Name) + if spec.Volumes[i].Name == preparedRolloutStateVolume { + return fmt.Errorf("prepared Agent rollout volume name %q is reserved", preparedRolloutStateVolume) } } - directoryOrCreate := corev1.HostPathDirectoryOrCreate - spec.Volumes = append(spec.Volumes, - corev1.Volume{ - Name: preparedRolloutLockVolume, - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ - Path: preparedRolloutLockDir, - Type: &directoryOrCreate, - }}, - }, - corev1.Volume{ - Name: preparedRolloutStateVolume, - VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, - }, - ) + spec.Volumes = append(spec.Volumes, corev1.Volume{ + Name: preparedRolloutStateVolume, + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + }) return nil } func configurePreparedContainer(container *corev1.Container) { - lockPath := preparedRolloutLockDir + "/" + container.Name + ".lock" statePath := preparedRolloutStateDir + "/" + container.Name + ".state" originalLiveness := container.LivenessProbe.DeepCopy() originalReadiness := container.ReadinessProbe.DeepCopy() if originalReadiness == nil { originalReadiness = originalLiveness } - setContainerEnv(container, rolloutEnabledEnv, "true") - setContainerEnv(container, rolloutLockPathEnv, lockPath) - setContainerEnv(container, rolloutStatePathEnv, statePath) - container.VolumeMounts = append(container.VolumeMounts, - corev1.VolumeMount{Name: preparedRolloutLockVolume, MountPath: preparedRolloutLockDir}, - corev1.VolumeMount{Name: preparedRolloutStateVolume, MountPath: preparedRolloutStateDir}, - ) + setContainerEnv(container, corev1.EnvVar{Name: rolloutEnabledEnv, Value: "true"}) + setContainerEnv(container, corev1.EnvVar{Name: rolloutStatePathEnv, Value: statePath}) + setContainerEnv(container, corev1.EnvVar{ + Name: rolloutPodUIDEnv, + ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "metadata.uid", + }}, + }) + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{Name: preparedRolloutStateVolume, MountPath: preparedRolloutStateDir}) container.StartupProbe = rolloutStateProbe(statePath, "prepared|activating|active", 1, 300) - container.LivenessProbe = rolloutHealthProbe(container.Name, statePath, true, originalLiveness) - container.ReadinessProbe = rolloutHealthProbe(container.Name, statePath, false, originalReadiness) + container.LivenessProbe = rolloutHealthProbe(container.Name, statePath, originalLiveness) + container.ReadinessProbe = rolloutHealthProbe(container.Name, statePath, originalReadiness) } -func setContainerEnv(container *corev1.Container, name, value string) { +func setContainerEnv(container *corev1.Container, env corev1.EnvVar) { for i := range container.Env { - if container.Env[i].Name == name { - container.Env[i] = corev1.EnvVar{Name: name, Value: value} + if container.Env[i].Name == env.Name { + container.Env[i] = env return } } - container.Env = append(container.Env, corev1.EnvVar{Name: name, Value: value}) + container.Env = append(container.Env, env) } func rolloutStateProbe(path, accepted string, period, failures int32) *corev1.Probe { - command := fmt.Sprintf(`case "$(cat %s 2>/dev/null)" in %s) exit 0;; *) exit 1;; esac`, path, accepted) + command := rolloutStateReadCommand(path) + fmt.Sprintf(`case "$state" in %s) exit 0;; *) exit 1;; esac`, accepted) return &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"sh", "-c", command}}}, PeriodSeconds: period, @@ -334,25 +284,21 @@ func rolloutStateProbe(path, accepted string, period, failures int32) *corev1.Pr } } -// rolloutHealthProbe accepts a sleeping Prepared process for liveness, but -// delegates to the component's real health mechanism after activation. This -// avoids both overlap false positives and permanently replacing Agent health -// with a state marker. -func rolloutHealthProbe(containerName, statePath string, allowWaiting bool, base *corev1.Probe) *corev1.Probe { - var activeHealth string +// rolloutHealthProbe deliberately treats Prepared as ready. That is the +// contract consumed by native DaemonSet maxSurge: images, init containers and +// the Agent graph are ready, while data-producing Fx hooks remain stopped. +func rolloutHealthProbe(containerName, statePath string, base *corev1.Probe) *corev1.Probe { + activeHealth := "exit 1" switch containerName { case string(apicommon.CoreAgentContainerName): activeHealth = "exec /opt/datadog-agent/bin/agent/agent health" case string(apicommon.TraceAgentContainerName): activeHealth = "exec 3<>/dev/tcp/127.0.0.1/8126; exec 3>&-; exec 3<&-" - default: - activeHealth = "exit 1" } - waiting := "" - if allowWaiting { - waiting = "prepared|activating) exit 0;; " - } - command := fmt.Sprintf(`state="$(cat %s 2>/dev/null)" || exit 1; case "$state" in %sactive) %s;; *) exit 1;; esac`, statePath, waiting, activeHealth) + // Prepared may wait indefinitely for the old Pod. Activating must use the + // normal liveness failure budget so a hung Fx start is eventually restarted. + acceptedWaiting := "prepared) exit 0;; " + command := rolloutStateReadCommand(statePath) + fmt.Sprintf(`case "$state" in %sactive) %s;; *) exit 1;; esac`, acceptedWaiting, activeHealth) probe := &corev1.Probe{PeriodSeconds: 10, TimeoutSeconds: 1, FailureThreshold: 3} if base != nil { @@ -362,188 +308,13 @@ func rolloutHealthProbe(containerName, statePath string, allowWaiting bool, base return probe } -type preparedHandoffCandidate struct { - replacement *corev1.Pod - old *corev1.Pod - nodeName string - reserved bool -} - -func (r *Reconciler) reconcilePreparedHandoff(ctx context.Context, ddai *datadoghqv1alpha1.DatadogAgentInternal, expectedDS *appsv1.DaemonSet, budgetValue intstr.IntOrString) (reconcile.Result, error) { - reader := r.apiReader - liveDS := &appsv1.DaemonSet{} - if err := reader.Get(ctx, client.ObjectKeyFromObject(expectedDS), liveDS); err != nil { - return reconcile.Result{}, client.IgnoreNotFound(err) - } - if !daemonSetControlledByDDAI(liveDS, ddai) || liveDS.Spec.Template.Annotations[preparedRolloutPhaseAnnotation] != preparedRolloutPhaseStandby || !resourceFallbackDaemonSetEligible(liveDS) { - return reconcile.Result{}, nil - } - currentRevision, err := currentDaemonSetRevision(ctx, reader, liveDS) - if err != nil || currentRevision == "" { - return reconcile.Result{}, err - } - pods, err := daemonSetPods(ctx, reader, liveDS) - if err != nil { - return reconcile.Result{}, err - } - budget, err := intstr.GetScaledValueFromIntOrPercent(&budgetValue, int(liveDS.Status.DesiredNumberScheduled), true) - if err != nil || budget <= 0 { - return reconcile.Result{}, err - } - candidates := preparedHandoffCandidates(liveDS, pods, currentRevision) - validReservations := make(map[string]struct{}, len(candidates)) - for _, candidate := range candidates { - if candidate.reserved { - validReservations[string(candidate.replacement.UID)] = struct{}{} - } - } - for i := range pods { - pod := &pods[i] - if pod.Spec.NodeName == "" || pod.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] != currentRevision || pod.Annotations[resourceFallbackOldPodAnnotation] == "" { - continue - } - if _, valid := validReservations[string(pod.UID)]; valid { - continue - } - return r.releasePreparedHandoffReservation(ctx, pod) - } - - consumed := consumedFallbackBudget(liveDS, pods, currentRevision, time.Now()) - if consumed > budget { - for i := len(candidates) - 1; i >= 0; i-- { - if candidates[i].reserved { - return r.releasePreparedHandoffReservation(ctx, candidates[i].replacement) - } - } - return reconcile.Result{}, nil - } - for _, candidate := range candidates { - if !candidate.reserved && consumed >= budget { - continue - } - if !candidate.reserved { - base := candidate.replacement.DeepCopy() - patched := candidate.replacement.DeepCopy() - if patched.Annotations == nil { - patched.Annotations = map[string]string{} - } - patched.Annotations[resourceFallbackOldPodAnnotation] = string(candidate.old.UID) - if err := r.client.Patch(ctx, patched, client.MergeFrom(base)); err != nil { - return reconcile.Result{}, fmt.Errorf("reserve prepared Agent handoff for Pod %s/%s: %w", patched.Namespace, patched.Name, err) - } - candidate.replacement = patched - consumed++ - } - liveCandidate, err := r.revalidatePreparedHandoff(ctx, liveDS, candidate, currentRevision) - if err != nil { - return reconcile.Result{}, err - } - if liveCandidate == nil { - continue - } - withinBudget, err := fallbackBudgetWithinLimit(ctx, reader, liveDS, budgetValue, currentRevision) - if err != nil { - return reconcile.Result{}, err - } - if !withinBudget { - return reconcile.Result{RequeueAfter: time.Second}, nil - } - uid := liveCandidate.old.UID - if err := r.client.Delete(ctx, liveCandidate.old, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { - return reconcile.Result{}, fmt.Errorf("delete old Agent Pod %s/%s for prepared handoff: %w", liveCandidate.old.Namespace, liveCandidate.old.Name, err) - } - if r.recorder != nil { - r.recorder.Eventf(ddai, corev1.EventTypeNormal, "AgentPreparedHandoff", "Deleted old Agent Pod %s on node %s after replacement %s reported Prepared", liveCandidate.old.Name, liveCandidate.nodeName, liveCandidate.replacement.Name) - } - return reconcile.Result{RequeueAfter: time.Second}, nil - } - return reconcile.Result{}, nil -} - -func (r *Reconciler) releasePreparedHandoffReservation(ctx context.Context, replacement *corev1.Pod) (reconcile.Result, error) { - base := replacement.DeepCopy() - patched := replacement.DeepCopy() - delete(patched.Annotations, resourceFallbackOldPodAnnotation) - if err := r.client.Patch(ctx, patched, client.MergeFrom(base)); err != nil { - return reconcile.Result{}, fmt.Errorf("release prepared Agent handoff reservation for Pod %s/%s: %w", patched.Namespace, patched.Name, err) - } - return reconcile.Result{RequeueAfter: time.Second}, nil -} - -func preparedHandoffCandidates(ds *appsv1.DaemonSet, pods []corev1.Pod, currentRevision string) []preparedHandoffCandidate { - oldByNode := map[string]*corev1.Pod{} - for i := range pods { - pod := &pods[i] - if pod.Spec.NodeName != "" && pod.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] != currentRevision && podAvailable(pod, ds.Spec.MinReadySeconds, time.Now()) { - oldByNode[pod.Spec.NodeName] = pod - } - } - var candidates []preparedHandoffCandidate - for i := range pods { - pod := &pods[i] - if pod.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] != currentRevision || !podPreparedForHandoff(pod) || podAvailable(pod, ds.Spec.MinReadySeconds, time.Now()) { - continue - } - old := oldByNode[pod.Spec.NodeName] - if old == nil { - continue - } - reservation := pod.Annotations[resourceFallbackOldPodAnnotation] - if reservation != "" && reservation != string(old.UID) { - continue - } - candidates = append(candidates, preparedHandoffCandidate{replacement: pod, old: old, nodeName: pod.Spec.NodeName, reserved: reservation != ""}) - } - sort.Slice(candidates, func(i, j int) bool { return candidates[i].nodeName < candidates[j].nodeName }) - return candidates -} - -func podPreparedForHandoff(pod *corev1.Pod) bool { - if pod.DeletionTimestamp != nil || pod.Status.Phase != corev1.PodRunning || len(pod.Status.InitContainerStatuses) != 2 || len(pod.Status.ContainerStatuses) != len(preparedRolloutContainerNames) { - return false - } - for i := range pod.Status.InitContainerStatuses { - status := &pod.Status.InitContainerStatuses[i] - if status.State.Terminated == nil || status.State.Terminated.ExitCode != 0 { - return false - } - } - seen := map[string]bool{} - for i := range pod.Status.ContainerStatuses { - status := &pod.Status.ContainerStatuses[i] - if status.Name != string(apicommon.CoreAgentContainerName) && status.Name != string(apicommon.TraceAgentContainerName) || status.State.Running == nil || status.Started == nil || !*status.Started || status.RestartCount != 0 { - return false - } - seen[status.Name] = true - } - return seen[string(apicommon.CoreAgentContainerName)] && seen[string(apicommon.TraceAgentContainerName)] +// rolloutStateReadCommand rejects state left by a previous container +// generation. EmptyDir volumes are Pod-lifetime, not container-lifetime, so +// every marker includes the writer PID and its Linux /proc start time. +func rolloutStateReadCommand(statePath string) string { + return fmt.Sprintf(`read -r state pid started extra < %s || exit 1; [ -z "$extra" ] || exit 1; case "$pid:$started" in *[!0-9:]*|:*|*:) exit 1;; esac; procstat="$(cat /proc/$pid/stat 2>/dev/null)" || exit 1; procstat="${procstat##*) }"; set -- $procstat; [ "$#" -ge 20 ] && [ "${20}" = "$started" ] || exit 1; `, statePath) } -func (r *Reconciler) revalidatePreparedHandoff(ctx context.Context, expectedDS *appsv1.DaemonSet, candidate preparedHandoffCandidate, expectedRevision string) (*preparedHandoffCandidate, error) { - liveDS := &appsv1.DaemonSet{} - if err := r.apiReader.Get(ctx, client.ObjectKeyFromObject(expectedDS), liveDS); err != nil { - return nil, client.IgnoreNotFound(err) - } - if liveDS.UID != expectedDS.UID || liveDS.Generation != expectedDS.Generation || liveDS.Spec.Template.Annotations[preparedRolloutPhaseAnnotation] != preparedRolloutPhaseStandby { - return nil, nil - } - revision, err := currentDaemonSetRevision(ctx, r.apiReader, liveDS) - if err != nil || revision != expectedRevision { - return nil, err - } - replacement := &corev1.Pod{} - old := &corev1.Pod{} - if err := r.apiReader.Get(ctx, client.ObjectKeyFromObject(candidate.replacement), replacement); err != nil { - return nil, client.IgnoreNotFound(err) - } - if err := r.apiReader.Get(ctx, client.ObjectKeyFromObject(candidate.old), old); err != nil { - return nil, client.IgnoreNotFound(err) - } - if replacement.UID != candidate.replacement.UID || old.UID != candidate.old.UID || !controlledByUID(replacement, liveDS.UID) || !controlledByUID(old, liveDS.UID) || replacement.Spec.NodeName != candidate.nodeName || old.Spec.NodeName != candidate.nodeName { - return nil, nil - } - if replacement.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] != revision || old.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] == revision || replacement.Annotations[resourceFallbackOldPodAnnotation] != string(old.UID) || !podPreparedForHandoff(replacement) || !podAvailable(old, liveDS.Spec.MinReadySeconds, time.Now()) { - return nil, nil - } - return &preparedHandoffCandidate{replacement: replacement, old: old, nodeName: candidate.nodeName, reserved: true}, nil +func hasRolloutMode(annotations map[string]string) bool { + return strings.EqualFold(annotations[preparedRolloutModeAnnotation], preparedRolloutModeV1) } diff --git a/internal/controller/datadogagentinternal/prepared_rollout_support.go b/internal/controller/datadogagentinternal/prepared_rollout_support.go new file mode 100644 index 0000000000..7abd185695 --- /dev/null +++ b/internal/controller/datadogagentinternal/prepared_rollout_support.go @@ -0,0 +1,229 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// Copyright 2016-present Datadog, Inc. + +package datadogagentinternal + +import ( + "context" + "encoding/json" + "fmt" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + datadoghqcommon "github.com/DataDog/datadog-operator/api/datadoghq/common" + datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" + datadoghqv2alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v2alpha1" + componentagent "github.com/DataDog/datadog-operator/internal/controller/datadogagent/component/agent" + "github.com/DataDog/datadog-operator/pkg/constants" +) + +const ( + defaultPreparedRolloutMaxUnavailable = 1 +) + +func configurePreparedSurge(ds *appsv1.DaemonSet, budget intstr.IntOrString) bool { + strategy := &ds.Spec.UpdateStrategy + if strategy.Type != "" && strategy.Type != appsv1.RollingUpdateDaemonSetStrategyType { + return false + } + if strategy.RollingUpdate == nil { + strategy.RollingUpdate = &appsv1.RollingUpdateDaemonSet{} + } + if _, err := intstr.GetScaledValueFromIntOrPercent(&budget, 100, true); err != nil { + return false + } + + strategy.Type = appsv1.RollingUpdateDaemonSetStrategyType + zero := intstr.FromInt(0) + strategy.RollingUpdate.MaxUnavailable = &zero + surge := budget + strategy.RollingUpdate.MaxSurge = &surge + return positiveIntOrPercent(&surge) +} + +func configureConventionalMigration(ds *appsv1.DaemonSet, budget intstr.IntOrString) { + zero := intstr.FromInt(0) + value := budget + ds.Spec.UpdateStrategy = appsv1.DaemonSetUpdateStrategy{ + Type: appsv1.RollingUpdateDaemonSetStrategyType, + RollingUpdate: &appsv1.RollingUpdateDaemonSet{ + MaxUnavailable: &value, + MaxSurge: &zero, + }, + } +} + +// prepareProfileAntiAffinityForSurge narrows the standard DAP anti-affinity so +// old and new revisions of the same profile may overlap. Unknown user-supplied +// anti-affinity fails closed. +func prepareProfileAntiAffinityForSurge(template *corev1.PodTemplateSpec) bool { + if template.Spec.Affinity == nil || template.Spec.Affinity.PodAntiAffinity == nil { + return true + } + if !apiequality.Semantic.DeepEqual(template.Spec.Affinity.PodAntiAffinity, broadAgentPodAntiAffinity()) { + return false + } + narrowed, ok := profileSurgePodAntiAffinity(template.Labels) + if !ok { + return false + } + template.Spec.Affinity.PodAntiAffinity = narrowed + return true +} + +func broadAgentPodAntiAffinity() *corev1.PodAntiAffinity { + return &corev1.PodAntiAffinity{RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: datadoghqcommon.AgentDeploymentComponentLabelKey, Operator: metav1.LabelSelectorOpIn, Values: []string{constants.DefaultAgentResourceSuffix}, + }}}, + TopologyKey: corev1.LabelHostname, + }}} +} + +func profileSurgePodAntiAffinity(podLabels map[string]string) (*corev1.PodAntiAffinity, bool) { + ddaName := podLabels[datadoghqcommon.AgentDeploymentNameLabelKey] + if ddaName == "" { + return nil, false + } + + profileRequirement := metav1.LabelSelectorRequirement{Key: constants.ProfileLabelKey} + if profileName := podLabels[constants.ProfileLabelKey]; profileName != "" { + profileRequirement.Operator = metav1.LabelSelectorOpNotIn + profileRequirement.Values = []string{profileName} + } else { + profileRequirement.Operator = metav1.LabelSelectorOpExists + } + componentRequirement := metav1.LabelSelectorRequirement{ + Key: datadoghqcommon.AgentDeploymentComponentLabelKey, Operator: metav1.LabelSelectorOpIn, Values: []string{constants.DefaultAgentResourceSuffix}, + } + + return &corev1.PodAntiAffinity{RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + componentRequirement, + {Key: datadoghqcommon.AgentDeploymentNameLabelKey, Operator: metav1.LabelSelectorOpIn, Values: []string{ddaName}}, + profileRequirement, + }}, + TopologyKey: corev1.LabelHostname, + }, + { + LabelSelector: &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + componentRequirement, + {Key: datadoghqcommon.AgentDeploymentNameLabelKey, Operator: metav1.LabelSelectorOpNotIn, Values: []string{ddaName}}, + }}, + TopologyKey: corev1.LabelHostname, + }, + }}, true +} + +func positiveIntOrPercent(value *intstr.IntOrString) bool { + if value == nil { + return false + } + scaled, err := intstr.GetScaledValueFromIntOrPercent(value, 100, true) + return err == nil && scaled > 0 +} + +func preparedRolloutBudget(ddai *datadoghqv1alpha1.DatadogAgentInternal, options *componentagent.ExtendedDaemonsetOptions) intstr.IntOrString { + if override, ok := ddai.Spec.Override[datadoghqv2alpha1.NodeAgentComponentName]; ok && override != nil && override.UpdateStrategy != nil && override.UpdateStrategy.RollingUpdate != nil && override.UpdateStrategy.RollingUpdate.MaxUnavailable != nil { + return *override.UpdateStrategy.RollingUpdate.MaxUnavailable + } + if options != nil && options.MaxPodUnavailable != "" { + return intstr.Parse(options.MaxPodUnavailable) + } + return intstr.FromInt(defaultPreparedRolloutMaxUnavailable) +} + +func daemonSetControlledByDDAI(ds *appsv1.DaemonSet, ddai *datadoghqv1alpha1.DatadogAgentInternal) bool { + owner := metav1.GetControllerOf(ds) + return owner != nil && owner.APIVersion == datadoghqv1alpha1.GroupVersion.String() && owner.Kind == "DatadogAgentInternal" && owner.UID == ddai.UID +} + +func preparedRolloutDaemonSetEligible(ds *appsv1.DaemonSet) bool { + if ds.DeletionTimestamp != nil || ds.Status.DesiredNumberScheduled <= 0 || ds.Status.ObservedGeneration != ds.Generation { + return false + } + return ds.Spec.UpdateStrategy.Type == appsv1.RollingUpdateDaemonSetStrategyType && + ds.Spec.UpdateStrategy.RollingUpdate != nil && positiveIntOrPercent(ds.Spec.UpdateStrategy.RollingUpdate.MaxSurge) +} + +func currentDaemonSetRevision(ctx context.Context, reader client.Reader, ds *appsv1.DaemonSet) (string, error) { + revisions := &appsv1.ControllerRevisionList{} + if err := reader.List(ctx, revisions, client.InNamespace(ds.Namespace)); err != nil { + return "", fmt.Errorf("list revisions for Agent DaemonSet %s/%s: %w", ds.Namespace, ds.Name, err) + } + var current *appsv1.ControllerRevision + for i := range revisions.Items { + revision := &revisions.Items[i] + if !controlledByUID(revision, ds.UID) { + continue + } + matches, err := controllerRevisionMatchesTemplate(revision, &ds.Spec.Template) + if err != nil { + return "", fmt.Errorf("decode revision %s for Agent DaemonSet %s/%s: %w", revision.Name, ds.Namespace, ds.Name, err) + } + if matches && (current == nil || revision.Revision > current.Revision) { + current = revision + } + } + if current == nil { + return "", nil + } + return current.Labels[appsv1.DefaultDaemonSetUniqueLabelKey], nil +} + +func controllerRevisionMatchesTemplate(revision *appsv1.ControllerRevision, template *corev1.PodTemplateSpec) (bool, error) { + var patch struct { + Spec struct { + Template corev1.PodTemplateSpec `json:"template"` + } `json:"spec"` + } + if err := json.Unmarshal(revision.Data.Raw, &patch); err != nil { + return false, err + } + return apiequality.Semantic.DeepEqual(patch.Spec.Template, *template), nil +} + +func daemonSetPods(ctx context.Context, reader client.Reader, ds *appsv1.DaemonSet) ([]corev1.Pod, error) { + selector, err := metav1.LabelSelectorAsSelector(ds.Spec.Selector) + if err != nil { + return nil, fmt.Errorf("build selector for Agent DaemonSet %s/%s: %w", ds.Namespace, ds.Name, err) + } + list := &corev1.PodList{} + if err := reader.List(ctx, list, client.InNamespace(ds.Namespace), client.MatchingLabelsSelector{Selector: selector}); err != nil { + return nil, fmt.Errorf("list Pods for Agent DaemonSet %s/%s: %w", ds.Namespace, ds.Name, err) + } + result := make([]corev1.Pod, 0, len(list.Items)) + for i := range list.Items { + if controlledByUID(&list.Items[i], ds.UID) { + result = append(result, list.Items[i]) + } + } + return result, nil +} + +func controlledByUID(obj metav1.Object, uid types.UID) bool { + owner := metav1.GetControllerOf(obj) + return owner != nil && owner.UID == uid +} + +func podAvailable(pod *corev1.Pod, minReadySeconds int32, now time.Time) bool { + if pod.DeletionTimestamp != nil || pod.Status.Phase != corev1.PodRunning { + return false + } + for i := range pod.Status.Conditions { + condition := &pod.Status.Conditions[i] + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { + return minReadySeconds == 0 || !condition.LastTransitionTime.IsZero() && condition.LastTransitionTime.Add(time.Duration(minReadySeconds)*time.Second).Before(now) + } + } + return false +} diff --git a/internal/controller/datadogagentinternal/prepared_rollout_test.go b/internal/controller/datadogagentinternal/prepared_rollout_test.go index 9311a3fde6..2777e0902a 100644 --- a/internal/controller/datadogagentinternal/prepared_rollout_test.go +++ b/internal/controller/datadogagentinternal/prepared_rollout_test.go @@ -5,842 +5,280 @@ package datadogagentinternal import ( - "context" - "errors" "strings" "testing" - "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - "sigs.k8s.io/controller-runtime/pkg/reconcile" apicommon "github.com/DataDog/datadog-operator/api/datadoghq/common" datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" + "github.com/DataDog/datadog-operator/pkg/constants" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestPrepareAgentTemplate(t *testing.T) { - arm := preparedRolloutDaemonSet() - require.NoError(t, prepareAgentTemplate(arm, preparedRolloutPhaseArm)) - assert.Equal(t, preparedRolloutPhaseArm, arm.Spec.Template.Annotations[preparedRolloutPhaseAnnotation]) - assert.NotEmpty(t, arm.Spec.Template.Spec.Containers[0].Ports, "arming keeps scheduler-visible ports because it does not overlap Pods") - for _, container := range arm.Spec.Template.Spec.Containers { - require.NotNil(t, container.StartupProbe) - require.NotNil(t, container.StartupProbe.Exec) - require.NotNil(t, container.ReadinessProbe) - require.NotNil(t, container.ReadinessProbe.Exec) - assert.Equal(t, "true", envValue(container.Env, rolloutEnabledEnv)) - assert.Contains(t, envValue(container.Env, rolloutLockPathEnv), container.Name+".lock") - assert.Contains(t, envValue(container.Env, rolloutStatePathEnv), container.Name+".state") - } - assert.Equal(t, "trace-agent", arm.Spec.Template.Spec.Containers[1].Command[0]) - assert.Contains(t, strings.Join(arm.Spec.Template.Spec.Containers[0].LivenessProbe.Exec.Command, " "), "agent health") - assert.Contains(t, strings.Join(arm.Spec.Template.Spec.Containers[1].LivenessProbe.Exec.Command, " "), "/dev/tcp/127.0.0.1/8126") - assert.NotContains(t, strings.Join(arm.Spec.Template.Spec.Containers[0].StartupProbe.Exec.Command, " "), "agent health", - "the sleeping replacement startup probe must not contact the old host-network listener") - - standby := preparedRolloutDaemonSet() - require.NoError(t, prepareAgentTemplate(standby, preparedRolloutPhaseStandby)) - for _, container := range standby.Spec.Template.Spec.Containers { - assert.Empty(t, container.Ports) - } -} - -func TestConfigurePreparedRolloutArmsThenStaysInStandby(t *testing.T) { - scheme := runtime.NewScheme() - require.NoError(t, appsv1.AddToScheme(scheme)) - reader := fake.NewClientBuilder().WithScheme(scheme).Build() - r := &Reconciler{apiReader: reader} - ddai := &datadoghqv1alpha1.DatadogAgentInternal{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{preparedRolloutAnnotation: "true"}}} - one := intstr.FromInt(1) - - desired := preparedRolloutDaemonSet() - phase, err := r.configurePreparedRollout(context.Background(), ddai, desired, one) - require.NoError(t, err) - assert.Equal(t, preparedRolloutPhaseArm, phase) - assert.Equal(t, intstr.FromInt(0), *desired.Spec.UpdateStrategy.RollingUpdate.MaxSurge) - - live := desired.DeepCopy() - // The API server defaults Pod fields on the live DaemonSet. Phase progress - // must use the controller-owned desired-template hash, not raw equality. - live.Spec.Template.Spec.DNSPolicy = corev1.DNSClusterFirst - live.UID = types.UID("daemonset-uid") - live.Generation = 2 - live.Status = appsv1.DaemonSetStatus{ - ObservedGeneration: 2, - DesiredNumberScheduled: 2, - UpdatedNumberScheduled: 2, - NumberReady: 2, - NumberAvailable: 2, - } - require.NoError(t, reader.Create(context.Background(), live)) - - next := preparedRolloutDaemonSet() - phase, err = r.configurePreparedRollout(context.Background(), ddai, next, one) - require.NoError(t, err) - assert.Equal(t, preparedRolloutPhaseStandby, phase) - assert.Equal(t, intstr.FromInt(1), *next.Spec.UpdateStrategy.RollingUpdate.MaxSurge) - assert.Equal(t, intstr.FromInt(0), *next.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable) - - live.Spec = next.Spec - live.Status.NumberReady = 1 - live.Status.NumberAvailable = 1 - require.NoError(t, reader.Update(context.Background(), live)) - - afterFailure := preparedRolloutDaemonSet() - phase, err = r.configurePreparedRollout(context.Background(), ddai, afterFailure, one) - require.NoError(t, err) - assert.Equal(t, preparedRolloutPhaseStandby, phase, "a mixed or failed rollout must not oscillate back to arm") -} - -func TestConfigurePreparedRolloutRejectsUnsupportedContainerWithoutMutation(t *testing.T) { - scheme := runtime.NewScheme() - require.NoError(t, appsv1.AddToScheme(scheme)) - r := &Reconciler{apiReader: fake.NewClientBuilder().WithScheme(scheme).Build()} - ddai := &datadoghqv1alpha1.DatadogAgentInternal{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{preparedRolloutAnnotation: "true"}}} - desired := preparedRolloutDaemonSet() - desired.Spec.Template.Spec.Containers = append(desired.Spec.Template.Spec.Containers, corev1.Container{Name: "security-agent"}) - original := desired.DeepCopy() - - _, err := r.configurePreparedRollout(context.Background(), ddai, desired, intstr.FromInt(1)) - require.Error(t, err) - assert.Equal(t, original, desired) -} - -func TestConfigurePreparedRolloutRejectsInvalidGates(t *testing.T) { - scheme := runtime.NewScheme() - reconciler := &Reconciler{apiReader: fake.NewClientBuilder().WithScheme(scheme).Build()} - enabled := &datadoghqv1alpha1.DatadogAgentInternal{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{preparedRolloutAnnotation: "true"}}} - - disabled := &datadoghqv1alpha1.DatadogAgentInternal{} - desired := preparedRolloutDaemonSet() - original := desired.DeepCopy() - phase, err := reconciler.configurePreparedRollout(context.Background(), disabled, desired, intstr.FromInt(1)) - require.NoError(t, err) - assert.Empty(t, phase) - assert.Equal(t, original, desired) - - _, err = reconciler.configurePreparedRollout(context.Background(), enabled, preparedRolloutDaemonSet(), intstr.FromInt(0)) - require.ErrorContains(t, err, "positive, valid maxUnavailable") - - onDelete := preparedRolloutDaemonSet() - onDelete.Spec.UpdateStrategy.Type = appsv1.OnDeleteDaemonSetStrategyType - _, err = reconciler.configurePreparedRollout(context.Background(), enabled, onDelete, intstr.FromInt(1)) - require.ErrorContains(t, err, "RollingUpdate strategy") +func TestPreparedRolloutRequiresExplicitMode(t *testing.T) { + ddai := &datadoghqv1alpha1.DatadogAgentInternal{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{}}} + assert.False(t, preparedRolloutEnabled(ddai)) + ddai.Annotations[preparedRolloutModeAnnotation] = "true" + assert.False(t, preparedRolloutEnabled(ddai)) + ddai.Annotations[preparedRolloutModeAnnotation] = preparedRolloutModeV1 + assert.True(t, preparedRolloutEnabled(ddai)) } -func TestConfigureArmStrategyInitializesRollingUpdate(t *testing.T) { - ds := &appsv1.DaemonSet{} - configureArmStrategy(ds, intstr.FromString("25%")) - require.NotNil(t, ds.Spec.UpdateStrategy.RollingUpdate) - assert.Equal(t, intstr.FromInt(0), *ds.Spec.UpdateStrategy.RollingUpdate.MaxSurge) - assert.Equal(t, intstr.FromString("25%"), *ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable) - - previous := ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable - configureArmStrategy(ds, intstr.FromInt(0)) - assert.Equal(t, previous, ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable) -} - -func TestPrepareAgentTemplateRejectsUnsafeTemplates(t *testing.T) { +func TestPrepareAgentTemplateNetworkingMatrix(t *testing.T) { tests := []struct { - name string - mutate func(*appsv1.DaemonSet) - wantErr string + name string + hostNetwork bool + hostPort int32 + wantError string + wantPortCount int }{ { - name: "host network disabled", - mutate: func(ds *appsv1.DaemonSet) { - ds.Spec.Template.Spec.HostNetwork = false - }, - wantErr: "hostNetwork=true", - }, - { - name: "windows pod os", - mutate: func(ds *appsv1.DaemonSet) { - ds.Spec.Template.Spec.OS = &corev1.PodOS{Name: corev1.Windows} - }, - wantErr: "Linux-only", - }, - { - name: "windows node selector", - mutate: func(ds *appsv1.DaemonSet) { - ds.Spec.Template.Spec.NodeSelector = map[string]string{corev1.LabelOSStable: "windows"} - }, - wantErr: "Linux-only", - }, - { - name: "legacy windows node selector", - mutate: func(ds *appsv1.DaemonSet) { - ds.Spec.Template.Spec.NodeSelector = map[string]string{"beta.kubernetes.io/os": "windows"} - }, - wantErr: "Linux-only", - }, - { - name: "missing trace container", - mutate: func(ds *appsv1.DaemonSet) { - ds.Spec.Template.Spec.Containers = ds.Spec.Template.Spec.Containers[:1] - }, - wantErr: "exactly agent and trace-agent", - }, - { - name: "unsupported container", - mutate: func(ds *appsv1.DaemonSet) { - ds.Spec.Template.Spec.Containers[1].Name = "security-agent" - }, - wantErr: "does not support container", - }, - { - name: "duplicate container", - mutate: func(ds *appsv1.DaemonSet) { - ds.Spec.Template.Spec.Containers[1].Name = string(apicommon.CoreAgentContainerName) - }, - wantErr: "duplicate container", - }, - { - name: "container lifecycle hook", - mutate: func(ds *appsv1.DaemonSet) { - ds.Spec.Template.Spec.Containers[0].Lifecycle = &corev1.Lifecycle{} - }, - wantErr: "lifecycle hooks", - }, - { - name: "container arguments", - mutate: func(ds *appsv1.DaemonSet) { - ds.Spec.Template.Spec.Containers[0].Args = []string{"--extra"} - }, - wantErr: "command arguments", - }, - { - name: "nonstandard core command", - mutate: func(ds *appsv1.DaemonSet) { - ds.Spec.Template.Spec.Containers[0].Command = []string{"agent", "start"} - }, - wantErr: "standard agent run command", - }, - { - name: "reserved mount path", - mutate: func(ds *appsv1.DaemonSet) { - ds.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: "custom", MountPath: preparedRolloutLockDir}} - }, - wantErr: "reserved name or path", - }, - { - name: "reserved mount name", - mutate: func(ds *appsv1.DaemonSet) { - ds.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: preparedRolloutLockVolume, MountPath: "/custom"}} - }, - wantErr: "reserved name or path", - }, - { - name: "unknown trace loader", - mutate: func(ds *appsv1.DaemonSet) { - ds.Spec.Template.Spec.Containers[1].Command = []string{"custom-loader"} - }, - wantErr: "unknown trace-agent loader", - }, - { - name: "unexpected init container", - mutate: func(ds *appsv1.DaemonSet) { - ds.Spec.Template.Spec.InitContainers[1].Name = "custom-init" - }, - wantErr: "does not support init container", - }, - { - name: "missing init container", - mutate: func(ds *appsv1.DaemonSet) { - ds.Spec.Template.Spec.InitContainers = ds.Spec.Template.Spec.InitContainers[:1] - }, - wantErr: "only init-volume and init-config", - }, - { - name: "duplicate init container", - mutate: func(ds *appsv1.DaemonSet) { - ds.Spec.Template.Spec.InitContainers[1].Name = string(apicommon.InitVolumeContainerName) - }, - wantErr: "requires init-volume and init-config", - }, - { - name: "init container lifecycle hook", - mutate: func(ds *appsv1.DaemonSet) { - ds.Spec.Template.Spec.InitContainers[0].Lifecycle = &corev1.Lifecycle{} - }, - wantErr: "ports or lifecycle hooks", + name: "pod network and UDS preserves declared container ports", + wantPortCount: 1, }, { - name: "init container port", - mutate: func(ds *appsv1.DaemonSet) { - ds.Spec.Template.Spec.InitContainers[0].Ports = []corev1.ContainerPort{{ContainerPort: 1234}} - }, - wantErr: "ports or lifecycle hooks", + name: "pod network and hostPort is rejected", + hostPort: 8126, + wantError: "cannot overlap Pod-networked containers that declare hostPort", }, { - name: "reserved volume name", - mutate: func(ds *appsv1.DaemonSet) { - ds.Spec.Template.Spec.Volumes = []corev1.Volume{{Name: preparedRolloutStateVolume}} - }, - wantErr: "reserved", - }, - { - name: "custom anti-affinity", - mutate: func(ds *appsv1.DaemonSet) { - ds.Spec.Template.Spec.Affinity = &corev1.Affinity{PodAntiAffinity: &corev1.PodAntiAffinity{}} - }, - wantErr: "custom Pod anti-affinity", + name: "host network strips scheduling port claims", + hostNetwork: true, + hostPort: 8126, + wantPortCount: 0, }, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ds := preparedRolloutDaemonSet() - test.mutate(ds) - err := prepareAgentTemplate(ds, preparedRolloutPhaseArm) - require.ErrorContains(t, err, test.wantErr) - }) - } -} - -func TestConfigurePreparedContainerReplacesRolloutEnvironment(t *testing.T) { - container := &corev1.Container{ - Name: "agent", - Env: []corev1.EnvVar{ - {Name: rolloutEnabledEnv, Value: "false"}, - {Name: "KEEP_ME", Value: "value"}, - }, - } - - configurePreparedContainer(container) - - assert.Equal(t, "true", envValue(container.Env, rolloutEnabledEnv)) - assert.Equal(t, "value", envValue(container.Env, "KEEP_ME")) - count := 0 - for i := range container.Env { - if container.Env[i].Name == rolloutEnabledEnv { - count++ - } - } - assert.Equal(t, 1, count, "rollout configuration must replace, not duplicate, an existing environment variable") -} - -func TestRequeuePreparedArm(t *testing.T) { - t.Run("polls while arm status is incomplete", func(t *testing.T) { - result := reconcile.Result{} - requeuePreparedArm(&result, preparedRolloutPhaseArm) - assert.Equal(t, time.Second, result.RequeueAfter) - }) - - t.Run("keeps an earlier requeue", func(t *testing.T) { - result := reconcile.Result{RequeueAfter: 500 * time.Millisecond} - requeuePreparedArm(&result, preparedRolloutPhaseArm) - assert.Equal(t, 500*time.Millisecond, result.RequeueAfter) - }) - - t.Run("does not poll once standby starts", func(t *testing.T) { - result := reconcile.Result{} - requeuePreparedArm(&result, preparedRolloutPhaseStandby) - assert.Zero(t, result.RequeueAfter) - }) -} - -func TestPodPreparedForHandoff(t *testing.T) { - pod := preparedReplacementPod() - assert.True(t, podPreparedForHandoff(pod)) - pod.Status.ContainerStatuses[1].Started = ptr.To(false) - assert.False(t, podPreparedForHandoff(pod)) - pod.Status.ContainerStatuses[1].Started = ptr.To(true) - pod.Status.ContainerStatuses[1].RestartCount = 1 - assert.False(t, podPreparedForHandoff(pod)) - - tests := []struct { - name string - mutate func(*corev1.Pod) - }{ - {name: "deleting", mutate: func(p *corev1.Pod) { now := metav1.Now(); p.DeletionTimestamp = &now }}, - {name: "not running", mutate: func(p *corev1.Pod) { p.Status.Phase = corev1.PodPending }}, - {name: "missing init status", mutate: func(p *corev1.Pod) { p.Status.InitContainerStatuses = p.Status.InitContainerStatuses[:1] }}, - {name: "missing container status", mutate: func(p *corev1.Pod) { p.Status.ContainerStatuses = p.Status.ContainerStatuses[:1] }}, - {name: "failed init", mutate: func(p *corev1.Pod) { p.Status.InitContainerStatuses[0].State.Terminated.ExitCode = 1 }}, - {name: "running init", mutate: func(p *corev1.Pod) { p.Status.InitContainerStatuses[0].State.Terminated = nil }}, - {name: "unknown container", mutate: func(p *corev1.Pod) { p.Status.ContainerStatuses[0].Name = "security-agent" }}, - {name: "container stopped", mutate: func(p *corev1.Pod) { p.Status.ContainerStatuses[0].State.Running = nil }}, - {name: "started unknown", mutate: func(p *corev1.Pod) { p.Status.ContainerStatuses[0].Started = nil }}, - {name: "duplicate agent", mutate: func(p *corev1.Pod) { p.Status.ContainerStatuses[1].Name = "agent" }}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - candidate := preparedReplacementPod() - test.mutate(candidate) - assert.False(t, podPreparedForHandoff(candidate)) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ds := preparedTestDaemonSet(tt.hostNetwork) + ds.Spec.Template.Spec.Containers[1].Ports[0].HostPort = tt.hostPort + err := prepareAgentTemplate(ds) + if tt.wantError != "" { + require.ErrorContains(t, err, tt.wantError) + return + } + require.NoError(t, err) + for i := range ds.Spec.Template.Spec.Containers { + assert.Len(t, ds.Spec.Template.Spec.Containers[i].Ports, tt.wantPortCount) + } + // UDS hostPath volumes are deliberately preserved; sleeping processes + // do not bind or unlink the shared pathname. + assert.True(t, hasHostPath(ds.Spec.Template.Spec.Volumes, "/var/run/datadog")) }) } } -func TestReconcilePreparedHandoffReservesThenDeletesOldPod(t *testing.T) { - scheme := runtime.NewScheme() - require.NoError(t, corev1.AddToScheme(scheme)) - require.NoError(t, appsv1.AddToScheme(scheme)) - require.NoError(t, datadoghqv1alpha1.AddToScheme(scheme)) - - ddai := &datadoghqv1alpha1.DatadogAgentInternal{ - ObjectMeta: metav1.ObjectMeta{Name: "agent", Namespace: "datadog-agent", UID: "ddai-uid", Annotations: map[string]string{preparedRolloutAnnotation: "true"}}, - } - ds := preparedRolloutDaemonSet() - ds.UID = "daemonset-uid" - ds.Generation = 2 - ds.OwnerReferences = []metav1.OwnerReference{{ - APIVersion: datadoghqv1alpha1.GroupVersion.String(), - Kind: "DatadogAgentInternal", - Name: ddai.Name, - UID: ddai.UID, - Controller: ptr.To(true), - }} - require.NoError(t, prepareAgentTemplate(ds, preparedRolloutPhaseStandby)) - require.True(t, configureResourceFallback(ds, intstr.FromInt(1))) - ds.Status = appsv1.DaemonSetStatus{ObservedGeneration: 2, DesiredNumberScheduled: 1} - - old := readyPod("old", "old-uid", "node-a", "old-revision", time.Now().Add(-time.Minute)) - old.Namespace = ds.Namespace - old.Labels["app"] = "agent" - old.OwnerReferences = []metav1.OwnerReference{daemonSetOwner(ds)} - replacement := preparedReplacementPod() - replacement.ObjectMeta = metav1.ObjectMeta{ - Name: "new", - Namespace: ds.Namespace, - UID: "new-uid", - Labels: map[string]string{ - "app": "agent", - appsv1.DefaultDaemonSetUniqueLabelKey: "new-revision", - }, - OwnerReferences: []metav1.OwnerReference{daemonSetOwner(ds)}, - } - replacement.Spec.NodeName = "node-a" - revision := controllerRevisionForTemplate(t, ds, "new-revision") - - c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(ddai, ds, old, replacement, revision).Build() - r := &Reconciler{client: c, apiReader: c} - result, err := r.reconcilePreparedHandoff(context.Background(), ddai, ds, intstr.FromInt(1)) - require.NoError(t, err) - assert.Equal(t, time.Second, result.RequeueAfter) - - err = c.Get(context.Background(), client.ObjectKeyFromObject(old), &corev1.Pod{}) - assert.True(t, apierrors.IsNotFound(err), "the exact old UID should be deleted after both replacement processes report Prepared") - updated := &corev1.Pod{} - require.NoError(t, c.Get(context.Background(), client.ObjectKeyFromObject(replacement), updated)) - assert.Equal(t, string(old.UID), updated.Annotations[resourceFallbackOldPodAnnotation]) -} - -func TestReconcilePreparedHandoffResumesReservedCandidateAtBudget(t *testing.T) { - fixture := newPreparedHandoffFixture(t) - fixture.replacement.Annotations = map[string]string{resourceFallbackOldPodAnnotation: string(fixture.old.UID)} - base := fixture.client(t, true, true, true) - r := &Reconciler{client: base, apiReader: base} - - result, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) +func TestConfigurePreparedRolloutUsesExistingBudget(t *testing.T) { + ddai := &datadoghqv1alpha1.DatadogAgentInternal{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ + preparedRolloutModeAnnotation: preparedRolloutModeV1, + }}} + ds := preparedTestDaemonSet(false) + ds.Spec.UpdateStrategy.RollingUpdate.MaxSurge = nil + budget := intstr.FromString("10%") + migrating, err := configurePreparedRollout(ddai, ds, nil, budget) require.NoError(t, err) - assert.Equal(t, time.Second, result.RequeueAfter) - err = base.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{}) - assert.True(t, apierrors.IsNotFound(err), "a restart after reserving the full budget must resume and delete the exact old Pod") + assert.False(t, migrating) + require.NotNil(t, ds.Spec.UpdateStrategy.RollingUpdate) + assert.Equal(t, intstr.FromString("10%"), *ds.Spec.UpdateStrategy.RollingUpdate.MaxSurge) + assert.Equal(t, intstr.FromInt(0), *ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable) + assert.Equal(t, preparedRolloutModeV1, ds.Spec.Template.Annotations[preparedRolloutModeAnnotation]) } -func TestReconcilePreparedHandoffFindsReservedCandidateAfterUnreservedAtBudget(t *testing.T) { - fixture := newPreparedHandoffFixture(t) - fixture.old.Spec.NodeName = "node-b" - fixture.replacement.Spec.NodeName = "node-b" - fixture.replacement.Annotations = map[string]string{resourceFallbackOldPodAnnotation: string(fixture.old.UID)} - - oldA := fixture.old.DeepCopy() - oldA.Name = "old-a" - oldA.UID = "old-a-uid" - oldA.Spec.NodeName = "node-a" - replacementA := fixture.replacement.DeepCopy() - replacementA.Name = "new-a" - replacementA.UID = "new-a-uid" - replacementA.Spec.NodeName = "node-a" - replacementA.Annotations = nil - - base := fixture.client(t, true, true, true) - require.NoError(t, base.Create(context.Background(), oldA)) - require.NoError(t, base.Create(context.Background(), replacementA)) - r := &Reconciler{client: base, apiReader: base} - - result, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) +func TestPreparedRolloutMigratesExistingProfileAntiAffinityBeforeSurge(t *testing.T) { + ddai := &datadoghqv1alpha1.DatadogAgentInternal{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ + preparedRolloutModeAnnotation: preparedRolloutModeV1, + }}} + budget := intstr.FromInt(1) + current := preparedTestDaemonSet(true) + current.Generation = 1 + current.Spec.Template.Spec.Affinity = &corev1.Affinity{PodAntiAffinity: broadAgentPodAntiAffinity()} + + desired := preparedTestDaemonSet(true) + migrating, err := configurePreparedRollout(ddai, desired, current, budget) require.NoError(t, err) - assert.Equal(t, time.Second, result.RequeueAfter) - err = base.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{}) - assert.True(t, apierrors.IsNotFound(err), "the reserved handoff must resume even when an unreserved node sorts first") - assertOldPodStillExists(t, base, oldA) -} - -func TestReconcilePreparedHandoffReleasesStaleReservation(t *testing.T) { - fixture := newPreparedHandoffFixture(t) - fixture.replacement.Annotations = map[string]string{resourceFallbackOldPodAnnotation: string(fixture.old.UID)} - fixture.replacement.Status.ContainerStatuses[0].RestartCount = 1 - base := fixture.client(t, true, true, true) - r := &Reconciler{client: base, apiReader: base} - - result, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.True(t, migrating) + assert.Equal(t, intstr.FromInt(0), *desired.Spec.UpdateStrategy.RollingUpdate.MaxSurge) + assert.Equal(t, budget, *desired.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable) + assert.False(t, hasRolloutMode(desired.Spec.Template.Annotations)) + expectedAffinity, ok := profileSurgePodAntiAffinity(desired.Spec.Template.Labels) + require.True(t, ok) + assert.Equal(t, expectedAffinity, desired.Spec.Template.Spec.Affinity.PodAntiAffinity) + assert.Nil(t, containerEnv(&desired.Spec.Template.Spec.Containers[0], rolloutEnabledEnv)) + + current = desired.DeepCopy() + current.Generation = 2 + current.Status = appsv1.DaemonSetStatus{ObservedGeneration: 1, DesiredNumberScheduled: 2, UpdatedNumberScheduled: 1, NumberAvailable: 1, NumberUnavailable: 1} + desired = preparedTestDaemonSet(true) + migrating, err = configurePreparedRollout(ddai, desired, current, budget) require.NoError(t, err) - assert.Equal(t, time.Second, result.RequeueAfter) - updated := &corev1.Pod{} - require.NoError(t, base.Get(context.Background(), client.ObjectKeyFromObject(fixture.replacement), updated)) - assert.Empty(t, updated.Annotations[resourceFallbackOldPodAnnotation], "an ineligible replacement must stop consuming rollout budget") - assertOldPodStillExists(t, base, fixture.old) -} - -func TestReconcilePreparedHandoffReleasesReservationAboveReducedBudget(t *testing.T) { - fixture := newPreparedHandoffFixture(t) - fixture.old.Spec.NodeName = "node-a" - fixture.replacement.Spec.NodeName = "node-a" - fixture.replacement.Annotations = map[string]string{resourceFallbackOldPodAnnotation: string(fixture.old.UID)} + require.True(t, migrating, "surge must wait until every old broad-affinity Pod is replaced") - oldB := fixture.old.DeepCopy() - oldB.Name = "old-b" - oldB.UID = "old-b-uid" - oldB.Spec.NodeName = "node-b" - replacementB := fixture.replacement.DeepCopy() - replacementB.Name = "new-b" - replacementB.UID = "new-b-uid" - replacementB.Spec.NodeName = "node-b" - replacementB.Annotations = map[string]string{resourceFallbackOldPodAnnotation: string(oldB.UID)} - - base := fixture.client(t, true, true, true) - require.NoError(t, base.Create(context.Background(), oldB)) - require.NoError(t, base.Create(context.Background(), replacementB)) - r := &Reconciler{client: base, apiReader: base} - - result, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + current.Status = appsv1.DaemonSetStatus{ObservedGeneration: 2, DesiredNumberScheduled: 2, UpdatedNumberScheduled: 2, NumberAvailable: 2} + desired = preparedTestDaemonSet(true) + migrating, err = configurePreparedRollout(ddai, desired, current, budget) require.NoError(t, err) - assert.Equal(t, time.Second, result.RequeueAfter) - updated := &corev1.Pod{} - require.NoError(t, base.Get(context.Background(), client.ObjectKeyFromObject(replacementB), updated)) - assert.Empty(t, updated.Annotations[resourceFallbackOldPodAnnotation], "one reservation must be released when the budget is reduced") - assertOldPodStillExists(t, base, fixture.old) - assertOldPodStillExists(t, base, oldB) + assert.False(t, migrating) + assert.Equal(t, budget, *desired.Spec.UpdateStrategy.RollingUpdate.MaxSurge) + assert.True(t, hasRolloutMode(desired.Spec.Template.Annotations)) } -func TestReconcilePreparedHandoffFailsClosed(t *testing.T) { - t.Run("API reader error", func(t *testing.T) { - fixture := newPreparedHandoffFixture(t) - base := fixture.client(t, true, true, true) - reader := interceptor.NewClient(base, interceptor.Funcs{ - Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { - return errors.New("read failed") - }, - }) - r := &Reconciler{client: base, apiReader: reader} - _, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) - require.ErrorContains(t, err, "read failed") - }) - - t.Run("foreign DaemonSet", func(t *testing.T) { - fixture := newPreparedHandoffFixture(t) - fixture.ds.OwnerReferences[0].UID = "other-ddai" - base := fixture.client(t, true, true, true) - r := &Reconciler{client: base, apiReader: base} - result, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) - require.NoError(t, err) - assert.Equal(t, reconcile.Result{}, result) - assertOldPodStillExists(t, base, fixture.old) - }) - - t.Run("missing current revision", func(t *testing.T) { - fixture := newPreparedHandoffFixture(t) - base := fixture.client(t, false, true, true) - r := &Reconciler{client: base, apiReader: base} - result, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) - require.NoError(t, err) - assert.Equal(t, reconcile.Result{}, result) - assertOldPodStillExists(t, base, fixture.old) - }) - - t.Run("revision list error", func(t *testing.T) { - fixture := newPreparedHandoffFixture(t) - base := fixture.client(t, true, true, true) - reader := interceptor.NewClient(base, interceptor.Funcs{ - List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { - if _, ok := list.(*appsv1.ControllerRevisionList); ok { - return errors.New("revision list failed") - } - return c.List(ctx, list, opts...) - }, - }) - r := &Reconciler{client: base, apiReader: reader} - _, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) - require.ErrorContains(t, err, "revision list failed") - assertOldPodStillExists(t, base, fixture.old) - }) - - t.Run("Pod list error", func(t *testing.T) { - fixture := newPreparedHandoffFixture(t) - base := fixture.client(t, true, true, true) - reader := interceptor.NewClient(base, interceptor.Funcs{ - List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { - if _, ok := list.(*corev1.PodList); ok { - return errors.New("Pod list failed") - } - return c.List(ctx, list, opts...) - }, - }) - r := &Reconciler{client: base, apiReader: reader} - _, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) - require.ErrorContains(t, err, "Pod list failed") - assertOldPodStillExists(t, base, fixture.old) - }) - - t.Run("invalid budget", func(t *testing.T) { - fixture := newPreparedHandoffFixture(t) - base := fixture.client(t, true, true, true) - r := &Reconciler{client: base, apiReader: base} - _, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromString("invalid")) - require.Error(t, err) - assertOldPodStillExists(t, base, fixture.old) - }) - - t.Run("budget already consumed", func(t *testing.T) { - fixture := newPreparedHandoffFixture(t) - fixture.ds.Status.NumberUnavailable = 1 - base := fixture.client(t, true, true, true) - r := &Reconciler{client: base, apiReader: base} - result, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) - require.NoError(t, err) - assert.Equal(t, reconcile.Result{}, result) - assertOldPodStillExists(t, base, fixture.old) - }) - - t.Run("mismatched reservation", func(t *testing.T) { - fixture := newPreparedHandoffFixture(t) - fixture.replacement.Annotations = map[string]string{resourceFallbackOldPodAnnotation: "another-old-pod"} - base := fixture.client(t, true, true, true) - r := &Reconciler{client: base, apiReader: base} - result, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) - require.NoError(t, err) - assert.Equal(t, time.Second, result.RequeueAfter) - updated := &corev1.Pod{} - require.NoError(t, base.Get(context.Background(), client.ObjectKeyFromObject(fixture.replacement), updated)) - assert.Empty(t, updated.Annotations[resourceFallbackOldPodAnnotation]) - assertOldPodStillExists(t, base, fixture.old) - }) - - t.Run("reservation patch error", func(t *testing.T) { - fixture := newPreparedHandoffFixture(t) - base := fixture.client(t, true, true, true) - writer := interceptor.NewClient(base, interceptor.Funcs{ - Patch: func(context.Context, client.WithWatch, client.Object, client.Patch, ...client.PatchOption) error { - return errors.New("patch failed") - }, - }) - r := &Reconciler{client: writer, apiReader: base} - _, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) - require.ErrorContains(t, err, "reserve prepared Agent handoff") - assertOldPodStillExists(t, base, fixture.old) - }) - - t.Run("old Pod delete error", func(t *testing.T) { - fixture := newPreparedHandoffFixture(t) - fixture.replacement.Annotations = map[string]string{resourceFallbackOldPodAnnotation: string(fixture.old.UID)} - base := fixture.client(t, true, true, true) - writer := interceptor.NewClient(base, interceptor.Funcs{ - Delete: func(context.Context, client.WithWatch, client.Object, ...client.DeleteOption) error { - return errors.New("delete failed") - }, - }) - r := &Reconciler{client: writer, apiReader: base} - _, err := r.reconcilePreparedHandoff(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(2)) - require.ErrorContains(t, err, "delete old Agent Pod") - assertOldPodStillExists(t, base, fixture.old) - }) -} - -func TestRevalidatePreparedHandoffRejectsStaleState(t *testing.T) { - t.Run("API reader error", func(t *testing.T) { - fixture := newPreparedHandoffFixture(t) - base := fixture.client(t, true, true, true) - reader := interceptor.NewClient(base, interceptor.Funcs{ - Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { - return errors.New("read failed") - }, - }) - r := &Reconciler{apiReader: reader} - candidate := fixture.candidate(true) - got, err := r.revalidatePreparedHandoff(context.Background(), fixture.ds, candidate, "new-revision") - require.ErrorContains(t, err, "read failed") - assert.Nil(t, got) - }) - - tests := []struct { - name string - mutateExpected func(*appsv1.DaemonSet) - mutateCandidate func(*preparedHandoffCandidate) - mutateObjects func(*preparedHandoffFixture) - includeReplacement bool - includeOld bool - expectedRevision string - }{ - {name: "DaemonSet UID changed", mutateExpected: func(ds *appsv1.DaemonSet) { ds.UID = "stale-daemonset" }, includeReplacement: true, includeOld: true, expectedRevision: "new-revision"}, - {name: "revision changed", includeReplacement: true, includeOld: true, expectedRevision: "other-revision"}, - {name: "replacement disappeared", includeOld: true, expectedRevision: "new-revision"}, - {name: "old Pod disappeared", includeReplacement: true, expectedRevision: "new-revision"}, - {name: "replacement UID changed", mutateCandidate: func(candidate *preparedHandoffCandidate) { candidate.replacement.UID = "stale-replacement" }, includeReplacement: true, includeOld: true, expectedRevision: "new-revision"}, - {name: "reservation changed", mutateObjects: func(fixture *preparedHandoffFixture) { - fixture.replacement.Annotations = map[string]string{resourceFallbackOldPodAnnotation: "different-old-pod"} - }, includeReplacement: true, includeOld: true, expectedRevision: "new-revision"}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - fixture := newPreparedHandoffFixture(t) - fixture.replacement.Annotations = map[string]string{resourceFallbackOldPodAnnotation: string(fixture.old.UID)} - if test.mutateObjects != nil { - test.mutateObjects(fixture) - } - base := fixture.client(t, true, test.includeReplacement, test.includeOld) - r := &Reconciler{apiReader: base} - expected := fixture.ds.DeepCopy() - if test.mutateExpected != nil { - test.mutateExpected(expected) - } - candidate := fixture.candidate(true) - if test.mutateCandidate != nil { - test.mutateCandidate(&candidate) - } - got, err := r.revalidatePreparedHandoff(context.Background(), expected, candidate, test.expectedRevision) - require.NoError(t, err) - assert.Nil(t, got) - }) +func TestPreparedReplacementReportsPreparedAsReady(t *testing.T) { + ds := preparedTestDaemonSet(false) + require.NoError(t, prepareAgentTemplate(ds)) + for i := range ds.Spec.Template.Spec.Containers { + container := &ds.Spec.Template.Spec.Containers[i] + require.NotNil(t, container.ReadinessProbe) + command := strings.Join(container.ReadinessProbe.Exec.Command, " ") + assert.Contains(t, command, "prepared) exit 0") + assert.NotContains(t, command, "prepared|activating") + assert.Contains(t, command, `/proc/$pid/stat`) + assert.Contains(t, command, `${20}`) + require.NotNil(t, container.LivenessProbe) + assert.NotContains(t, strings.Join(container.LivenessProbe.Exec.Command, " "), "prepared|activating") + assert.Contains(t, strings.Join(container.LivenessProbe.Exec.Command, " "), "prepared) exit 0") + + uidEnv := containerEnv(container, rolloutPodUIDEnv) + require.NotNil(t, uidEnv) + require.NotNil(t, uidEnv.ValueFrom) + require.NotNil(t, uidEnv.ValueFrom.FieldRef) + assert.Equal(t, "metadata.uid", uidEnv.ValueFrom.FieldRef.FieldPath) + stateEnv := containerEnv(container, rolloutStatePathEnv) + require.NotNil(t, stateEnv) + assert.True(t, strings.HasPrefix(stateEnv.Value, preparedRolloutStateDir+"/")) } } -type preparedHandoffFixture struct { - scheme *runtime.Scheme - ddai *datadoghqv1alpha1.DatadogAgentInternal - ds *appsv1.DaemonSet - old *corev1.Pod - replacement *corev1.Pod - revision *appsv1.ControllerRevision -} - -func newPreparedHandoffFixture(t *testing.T) *preparedHandoffFixture { - t.Helper() - scheme := runtime.NewScheme() - require.NoError(t, corev1.AddToScheme(scheme)) - require.NoError(t, appsv1.AddToScheme(scheme)) - require.NoError(t, datadoghqv1alpha1.AddToScheme(scheme)) - - ddai := &datadoghqv1alpha1.DatadogAgentInternal{ObjectMeta: metav1.ObjectMeta{ - Name: "agent", Namespace: "datadog-agent", UID: "ddai-uid", Annotations: map[string]string{preparedRolloutAnnotation: "true"}, - }} - ds := preparedRolloutDaemonSet() - ds.UID = "daemonset-uid" - ds.Generation = 2 - ds.OwnerReferences = []metav1.OwnerReference{{ - APIVersion: datadoghqv1alpha1.GroupVersion.String(), Kind: "DatadogAgentInternal", Name: ddai.Name, UID: ddai.UID, Controller: ptr.To(true), - }} - require.NoError(t, prepareAgentTemplate(ds, preparedRolloutPhaseStandby)) - require.True(t, configureResourceFallback(ds, intstr.FromInt(1))) - ds.Status = appsv1.DaemonSetStatus{ObservedGeneration: 2, DesiredNumberScheduled: 1} - - old := readyPod("old", "old-uid", "node-a", "old-revision", time.Now().Add(-time.Minute)) - old.Namespace = ds.Namespace - old.Labels["app"] = "agent" - old.OwnerReferences = []metav1.OwnerReference{daemonSetOwner(ds)} - replacement := preparedReplacementPod() - replacement.ObjectMeta = metav1.ObjectMeta{ - Name: "new", Namespace: ds.Namespace, UID: "new-uid", - Labels: map[string]string{"app": "agent", appsv1.DefaultDaemonSetUniqueLabelKey: "new-revision"}, - OwnerReferences: []metav1.OwnerReference{daemonSetOwner(ds)}, +func TestPreparedRolloutStateUsesReservedPrivateEmptyDir(t *testing.T) { + ds := preparedTestDaemonSet(false) + require.NoError(t, prepareAgentTemplate(ds)) + volume := findVolumeByName(ds.Spec.Template.Spec.Volumes, preparedRolloutStateVolume) + require.NotNil(t, volume) + require.NotNil(t, volume.EmptyDir) + for i := range ds.Spec.Template.Spec.Containers { + assert.Contains(t, ds.Spec.Template.Spec.Containers[i].VolumeMounts, corev1.VolumeMount{Name: preparedRolloutStateVolume, MountPath: preparedRolloutStateDir}) } - replacement.Spec.NodeName = "node-a" - return &preparedHandoffFixture{ - scheme: scheme, ddai: ddai, ds: ds, old: old, replacement: replacement, - revision: controllerRevisionForTemplate(t, ds, "new-revision"), - } + conflicting := preparedTestDaemonSet(false) + conflicting.Spec.Template.Spec.Volumes = append(conflicting.Spec.Template.Spec.Volumes, corev1.Volume{Name: "shared", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run"}}}) + conflicting.Spec.Template.Spec.Containers[0].VolumeMounts = append(conflicting.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{Name: "shared", MountPath: "/var/run"}) + require.ErrorContains(t, prepareAgentTemplate(conflicting), "reserved name or path") } -func (f *preparedHandoffFixture) client(t *testing.T, includeRevision, includeReplacement, includeOld bool) client.WithWatch { - t.Helper() - objects := []client.Object{f.ddai, f.ds} - if includeRevision { - objects = append(objects, f.revision) - } - if includeReplacement { - objects = append(objects, f.replacement) - } - if includeOld { - objects = append(objects, f.old) - } - return fake.NewClientBuilder().WithScheme(f.scheme).WithObjects(objects...).Build() +func TestPreparedRolloutRejectsUngatedComponents(t *testing.T) { + ds := preparedTestDaemonSet(false) + ds.Spec.Template.Spec.Containers = append(ds.Spec.Template.Spec.Containers, corev1.Container{Name: "system-probe"}) + require.ErrorContains(t, prepareAgentTemplate(ds), "supports exactly agent and trace-agent") } -func (f *preparedHandoffFixture) candidate(reserved bool) preparedHandoffCandidate { - return preparedHandoffCandidate{replacement: f.replacement.DeepCopy(), old: f.old.DeepCopy(), nodeName: "node-a", reserved: reserved} -} - -func assertOldPodStillExists(t *testing.T, c client.Client, old *corev1.Pod) { - t.Helper() - require.NoError(t, c.Get(context.Background(), client.ObjectKeyFromObject(old), &corev1.Pod{})) -} +func TestProfileSurgeAntiAffinityAllowsOnlySameProfileOverlap(t *testing.T) { + antiAffinity, ok := profileSurgePodAntiAffinity(map[string]string{ + apicommon.AgentDeploymentNameLabelKey: "agent-a", + constants.ProfileLabelKey: "blue", + }) + require.True(t, ok) + require.Len(t, antiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, 2) -func preparedReplacementPod() *corev1.Pod { - return &corev1.Pod{Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - InitContainerStatuses: []corev1.ContainerStatus{ - {Name: "init-volume", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 0}}}, - {Name: "init-config", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 0}}}, - }, - ContainerStatuses: []corev1.ContainerStatus{ - {Name: "agent", Started: ptr.To(true), State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}, - {Name: "trace-agent", Started: ptr.To(true), State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}, - }, - }} + blocked := func(podLabels map[string]string) bool { + for _, term := range antiAffinity.RequiredDuringSchedulingIgnoredDuringExecution { + selector, err := metav1.LabelSelectorAsSelector(term.LabelSelector) + require.NoError(t, err) + if selector.Matches(labels.Set(podLabels)) { + return true + } + } + return false + } + base := map[string]string{apicommon.AgentDeploymentComponentLabelKey: constants.DefaultAgentResourceSuffix} + assert.False(t, blocked(mergeLabels(base, map[string]string{apicommon.AgentDeploymentNameLabelKey: "agent-a", constants.ProfileLabelKey: "blue"}))) + assert.True(t, blocked(mergeLabels(base, map[string]string{apicommon.AgentDeploymentNameLabelKey: "agent-a", constants.ProfileLabelKey: "green"}))) + assert.True(t, blocked(mergeLabels(base, map[string]string{apicommon.AgentDeploymentNameLabelKey: "agent-b", constants.ProfileLabelKey: "blue"}))) } -func preparedRolloutDaemonSet() *appsv1.DaemonSet { - one := intstr.FromInt(1) +func preparedTestDaemonSet(hostNetwork bool) *appsv1.DaemonSet { + port := corev1.ContainerPort{Name: "declared", ContainerPort: 8126, Protocol: corev1.ProtocolTCP} return &appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{Name: "datadog-agent", Namespace: "datadog-agent"}, + ObjectMeta: metav1.ObjectMeta{Name: "agent", Namespace: "default"}, Spec: appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "agent"}}, - UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ - Type: appsv1.RollingUpdateDaemonSetStrategyType, - RollingUpdate: &appsv1.RollingUpdateDaemonSet{ - MaxUnavailable: &one, - MaxSurge: &one, - }, - }, Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "agent"}}, + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{ + "app": "agent", + apicommon.AgentDeploymentNameLabelKey: "agent", + apicommon.AgentDeploymentComponentLabelKey: constants.DefaultAgentResourceSuffix, + }}, Spec: corev1.PodSpec{ - HostNetwork: true, + HostNetwork: hostNetwork, + NodeSelector: map[string]string{corev1.LabelOSStable: "linux"}, InitContainers: []corev1.Container{ - {Name: "init-volume"}, - {Name: "init-config"}, + {Name: string(apicommon.InitVolumeContainerName)}, + {Name: string(apicommon.InitConfigContainerName)}, }, Containers: []corev1.Container{ - {Name: "agent", Command: []string{"agent", "run"}, Ports: []corev1.ContainerPort{{ContainerPort: 8125}}}, - {Name: "trace-agent", Command: []string{"trace-loader", "/etc/datadog-agent/datadog.yaml", "trace-agent", "--config=/etc/datadog-agent/datadog.yaml"}, Ports: []corev1.ContainerPort{{ContainerPort: 8126}}}, + {Name: string(apicommon.CoreAgentContainerName), Command: []string{"agent", "run"}, Ports: []corev1.ContainerPort{port}, LivenessProbe: &corev1.Probe{}}, + {Name: string(apicommon.TraceAgentContainerName), Command: []string{"/entrypoint.sh", "trace-agent"}, Ports: []corev1.ContainerPort{port}, LivenessProbe: &corev1.Probe{}}, }, + Volumes: []corev1.Volume{{ + Name: "sockets", + VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/run/datadog", + Type: ptr.To(corev1.HostPathDirectoryOrCreate), + }}, + }}, + }, + }, + UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ + Type: appsv1.RollingUpdateDaemonSetStrategyType, + RollingUpdate: &appsv1.RollingUpdateDaemonSet{ + MaxUnavailable: ptr.To(intstr.FromInt(1)), + MaxSurge: ptr.To(intstr.FromInt(1)), }, }, }, } } -func envValue(env []corev1.EnvVar, name string) string { - for i := range env { - if env[i].Name == name { - return env[i].Value +func hasHostPath(volumes []corev1.Volume, path string) bool { + for i := range volumes { + if volumes[i].HostPath != nil && volumes[i].HostPath.Path == path { + return true } } - return "" + return false +} + +func findVolumeByName(volumes []corev1.Volume, name string) *corev1.Volume { + for i := range volumes { + if volumes[i].Name == name { + return &volumes[i] + } + } + return nil +} + +func containerEnv(container *corev1.Container, name string) *corev1.EnvVar { + for i := range container.Env { + if container.Env[i].Name == name { + return &container.Env[i] + } + } + return nil +} + +func mergeLabels(left, right map[string]string) map[string]string { + result := make(map[string]string, len(left)+len(right)) + for key, value := range left { + result[key] = value + } + for key, value := range right { + result[key] = value + } + return result } diff --git a/internal/controller/datadogagentinternal/resource_fallback.go b/internal/controller/datadogagentinternal/resource_fallback.go index 99f5a100bf..b6b13c62c3 100644 --- a/internal/controller/datadogagentinternal/resource_fallback.go +++ b/internal/controller/datadogagentinternal/resource_fallback.go @@ -6,7 +6,6 @@ package datadogagentinternal import ( "context" - "encoding/json" "fmt" "slices" "sort" @@ -21,7 +20,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" resourcehelper "k8s.io/component-helpers/resource" "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" @@ -29,131 +27,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - datadoghqcommon "github.com/DataDog/datadog-operator/api/datadoghq/common" datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" - datadoghqv2alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v2alpha1" - componentagent "github.com/DataDog/datadog-operator/internal/controller/datadogagent/component/agent" - "github.com/DataDog/datadog-operator/pkg/constants" ) -const ( - resourceFallbackOldPodAnnotation = "agent.datadoghq.com/resource-fallback-old-pod-uid" - apiPodNodeNameField = "spec.nodeName" - defaultFallbackMaxUnavailable = 1 -) - -// configureResourceFallback keeps surge opt-in. When it is requested, the -// existing maxUnavailable setting becomes both the surge limit and the -// Operator's resource-pressure fallback budget. The emitted maxUnavailable is -// intentionally 0 so the native DaemonSet controller never proactively -// deletes an old Pod; only the resource-proven fallback below may do that. -func configureResourceFallback(ds *appsv1.DaemonSet, budget intstr.IntOrString) bool { - strategy := &ds.Spec.UpdateStrategy - if strategy.Type != "" && strategy.Type != appsv1.RollingUpdateDaemonSetStrategyType { - return false - } - if strategy.RollingUpdate == nil || !positiveIntOrPercent(strategy.RollingUpdate.MaxSurge) { - return false - } - if _, err := intstr.GetScaledValueFromIntOrPercent(&budget, 100, true); err != nil { - return false - } - - strategy.Type = appsv1.RollingUpdateDaemonSetStrategyType - zero := intstr.FromInt(0) - strategy.RollingUpdate.MaxUnavailable = &zero - if positiveIntOrPercent(&budget) { - surge := budget - strategy.RollingUpdate.MaxSurge = &surge - return true - } - return false -} - -// prepareProfileAntiAffinityForSurge narrows the standard DAP anti-affinity so -// only old and new revisions of the same profile and DDA may overlap. Unknown -// or user-supplied anti-affinity fails closed. -func prepareProfileAntiAffinityForSurge(template *corev1.PodTemplateSpec) bool { - if template.Spec.Affinity == nil || template.Spec.Affinity.PodAntiAffinity == nil { - return true - } - if !apiequality.Semantic.DeepEqual(template.Spec.Affinity.PodAntiAffinity, broadAgentPodAntiAffinity()) { - return false - } - narrowed, ok := profileSurgePodAntiAffinity(template.Labels) - if !ok { - return false - } - template.Spec.Affinity.PodAntiAffinity = narrowed - return true -} - -func broadAgentPodAntiAffinity() *corev1.PodAntiAffinity { - return &corev1.PodAntiAffinity{RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{{ - Key: datadoghqcommon.AgentDeploymentComponentLabelKey, - Operator: metav1.LabelSelectorOpIn, - Values: []string{constants.DefaultAgentResourceSuffix}, - }}}, - TopologyKey: corev1.LabelHostname, - }}} -} - -func profileSurgePodAntiAffinity(podLabels map[string]string) (*corev1.PodAntiAffinity, bool) { - ddaName := podLabels[datadoghqcommon.AgentDeploymentNameLabelKey] - if ddaName == "" { - return nil, false - } - - profileRequirement := metav1.LabelSelectorRequirement{Key: constants.ProfileLabelKey} - if profileName := podLabels[constants.ProfileLabelKey]; profileName != "" { - profileRequirement.Operator = metav1.LabelSelectorOpNotIn - profileRequirement.Values = []string{profileName} - } else { - profileRequirement.Operator = metav1.LabelSelectorOpExists - } - componentRequirement := metav1.LabelSelectorRequirement{ - Key: datadoghqcommon.AgentDeploymentComponentLabelKey, - Operator: metav1.LabelSelectorOpIn, - Values: []string{constants.DefaultAgentResourceSuffix}, - } - - return &corev1.PodAntiAffinity{RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ - componentRequirement, - {Key: datadoghqcommon.AgentDeploymentNameLabelKey, Operator: metav1.LabelSelectorOpIn, Values: []string{ddaName}}, - profileRequirement, - }}, - TopologyKey: corev1.LabelHostname, - }, - { - LabelSelector: &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ - componentRequirement, - {Key: datadoghqcommon.AgentDeploymentNameLabelKey, Operator: metav1.LabelSelectorOpNotIn, Values: []string{ddaName}}, - }}, - TopologyKey: corev1.LabelHostname, - }, - }}, true -} - -func positiveIntOrPercent(value *intstr.IntOrString) bool { - if value == nil { - return false - } - scaled, err := intstr.GetScaledValueFromIntOrPercent(value, 100, true) - return err == nil && scaled > 0 -} - -func resourceFallbackBudget(ddai *datadoghqv1alpha1.DatadogAgentInternal, options *componentagent.ExtendedDaemonsetOptions) intstr.IntOrString { - if override, ok := ddai.Spec.Override[datadoghqv2alpha1.NodeAgentComponentName]; ok && override != nil && override.UpdateStrategy != nil && override.UpdateStrategy.RollingUpdate != nil && override.UpdateStrategy.RollingUpdate.MaxUnavailable != nil { - return *override.UpdateStrategy.RollingUpdate.MaxUnavailable - } - if options != nil && options.MaxPodUnavailable != "" { - return intstr.Parse(options.MaxPodUnavailable) - } - return intstr.FromInt(defaultFallbackMaxUnavailable) -} +const resourceFallbackOldPodAnnotation = "agent.datadoghq.com/resource-fallback-old-pod-uid" type resourceShortage struct { cpu bool @@ -168,65 +45,55 @@ type fallbackCandidate struct { reserved bool } +// reconcileResourceFallback breaks the maxSurge capacity deadlock only when +// the scheduler reports CPU and/or memory as the sole target-node blocker and +// removing the exact old Agent Pod is sufficient to make the replacement fit. +// The original maxUnavailable value is reused as the deletion budget. func (r *Reconciler) reconcileResourceFallback(ctx context.Context, ddai *datadoghqv1alpha1.DatadogAgentInternal, expectedDS *appsv1.DaemonSet, budgetValue intstr.IntOrString) (reconcile.Result, error) { reader := r.apiReader if reader == nil { reader = r.client } - ds := &appsv1.DaemonSet{} - key := client.ObjectKeyFromObject(expectedDS) - if err := reader.Get(ctx, key, ds); err != nil { - if apierrors.IsNotFound(err) { - return reconcile.Result{}, nil - } - return reconcile.Result{}, fmt.Errorf("get Agent DaemonSet for resource fallback: %w", err) + if err := reader.Get(ctx, client.ObjectKeyFromObject(expectedDS), ds); err != nil { + return reconcile.Result{}, client.IgnoreNotFound(err) } - if !daemonSetControlledByDDAI(ds, ddai) || !resourceFallbackDaemonSetEligible(ds) { + if !daemonSetControlledByDDAI(ds, ddai) || !preparedRolloutDaemonSetEligible(ds) || !hasRolloutMode(ds.Spec.Template.Annotations) { return reconcile.Result{}, nil } - - desired := int(ds.Status.DesiredNumberScheduled) - budget, err := intstr.GetScaledValueFromIntOrPercent(&budgetValue, desired, true) + budget, err := intstr.GetScaledValueFromIntOrPercent(&budgetValue, int(ds.Status.DesiredNumberScheduled), true) if err != nil { return reconcile.Result{}, fmt.Errorf("resolve Agent resource fallback budget: %w", err) } if budget <= 0 { return reconcile.Result{}, nil } - - currentRevision, err := currentDaemonSetRevision(ctx, reader, ds) - if err != nil { + revision, err := currentDaemonSetRevision(ctx, reader, ds) + if err != nil || revision == "" { return reconcile.Result{}, err } - if currentRevision == "" { - return reconcile.Result{}, nil - } - pods, err := daemonSetPods(ctx, reader, ds) if err != nil { return reconcile.Result{}, err } - consumed := consumedFallbackBudget(ds, pods, currentRevision, time.Now()) - candidates := fallbackCandidates(ds, pods, currentRevision, time.Now()) + consumed := consumedResourceFallbackBudget(ds, pods, revision, time.Now()) if consumed > budget { return reconcile.Result{}, nil } - for _, candidate := range candidates { + for _, candidate := range fallbackCandidates(ds, pods, revision, time.Now()) { if !candidate.reserved && consumed >= budget { break } - if !candidate.reserved { - liveCandidate, err := r.revalidateFallbackCandidate(ctx, reader, ds, candidate, currentRevision, false) + live, err := r.revalidateFallbackCandidate(ctx, reader, ds, candidate, revision, false) if err != nil { return reconcile.Result{}, err } - if liveCandidate == nil { + if live == nil { continue } - candidate = *liveCandidate + candidate = *live base := candidate.pending.DeepCopy() patched := candidate.pending.DeepCopy() if patched.Annotations == nil { @@ -238,176 +105,71 @@ func (r *Reconciler) reconcileResourceFallback(ctx context.Context, ddai *datado } candidate.pending = patched candidate.reserved = true - consumed++ } - liveCandidate, err := r.revalidateFallbackCandidate(ctx, reader, ds, candidate, currentRevision, true) + live, err := r.revalidateFallbackCandidate(ctx, reader, ds, candidate, revision, true) if err != nil { return reconcile.Result{}, err } - if liveCandidate == nil { + if live == nil { continue } - withinBudget, err := fallbackBudgetWithinLimit(ctx, reader, ds, budgetValue, currentRevision) + withinBudget, err := resourceFallbackBudgetWithinLimit(ctx, reader, ds, budgetValue, revision) if err != nil { return reconcile.Result{}, err } if !withinBudget { return reconcile.Result{RequeueAfter: time.Second}, nil } - - uid := liveCandidate.old.UID - if err := r.client.Delete(ctx, liveCandidate.old, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { - return reconcile.Result{}, fmt.Errorf("delete old Agent Pod %s/%s for resource fallback: %w", liveCandidate.old.Namespace, liveCandidate.old.Name, err) + uid := live.old.UID + if err := r.client.Delete(ctx, live.old, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("delete old Agent Pod %s/%s for resource fallback: %w", live.old.Namespace, live.old.Name, err) } - - logger := ctrl.LoggerFrom(ctx).WithValues("daemonset", ds.Name, "node", liveCandidate.nodeName, "oldPod", liveCandidate.old.Name, "replacementPod", liveCandidate.pending.Name) - logger.Info("Deleted old Agent Pod after proving the surged replacement was blocked only by node CPU or memory") + ctrl.LoggerFrom(ctx).WithValues("daemonset", ds.Name, "node", live.nodeName, "oldPod", live.old.Name, "replacementPod", live.pending.Name).Info("Deleted old Agent Pod after proving the surged replacement was blocked only by node CPU or memory") if r.recorder != nil { - r.recorder.Eventf(ddai, corev1.EventTypeWarning, "AgentResourceFallback", "Deleted old Agent Pod %s on node %s because replacement %s could not fit alongside it", liveCandidate.old.Name, liveCandidate.nodeName, liveCandidate.pending.Name) + r.recorder.Eventf(ddai, corev1.EventTypeWarning, "AgentResourceFallback", "Deleted old Agent Pod %s on node %s because replacement %s could not fit alongside it", live.old.Name, live.nodeName, live.pending.Name) } return reconcile.Result{RequeueAfter: time.Second}, nil } - return reconcile.Result{}, nil } -func fallbackBudgetWithinLimit(ctx context.Context, reader client.Reader, expectedDS *appsv1.DaemonSet, budgetValue intstr.IntOrString, expectedRevision string) (bool, error) { - liveDS := &appsv1.DaemonSet{} - if err := reader.Get(ctx, client.ObjectKeyFromObject(expectedDS), liveDS); err != nil { - return false, client.IgnoreNotFound(err) - } - if liveDS.UID != expectedDS.UID || liveDS.Generation != expectedDS.Generation || !resourceFallbackDaemonSetEligible(liveDS) { - return false, nil - } - revision, err := currentDaemonSetRevision(ctx, reader, liveDS) - if err != nil || revision != expectedRevision { - return false, err - } - pods, err := daemonSetPods(ctx, reader, liveDS) - if err != nil { - return false, err - } - budget, err := intstr.GetScaledValueFromIntOrPercent(&budgetValue, int(liveDS.Status.DesiredNumberScheduled), true) - if err != nil || budget <= 0 { - return false, err - } - return consumedFallbackBudget(liveDS, pods, revision, time.Now()) <= budget, nil -} - -func daemonSetControlledByDDAI(ds *appsv1.DaemonSet, ddai *datadoghqv1alpha1.DatadogAgentInternal) bool { - owner := metav1.GetControllerOf(ds) - return owner != nil && owner.APIVersion == datadoghqv1alpha1.GroupVersion.String() && owner.Kind == "DatadogAgentInternal" && owner.UID == ddai.UID -} - -func resourceFallbackDaemonSetEligible(ds *appsv1.DaemonSet) bool { - if ds.DeletionTimestamp != nil || ds.Status.DesiredNumberScheduled <= 0 || ds.Status.ObservedGeneration != ds.Generation { - return false - } - if ds.Spec.UpdateStrategy.Type != appsv1.RollingUpdateDaemonSetStrategyType || ds.Spec.UpdateStrategy.RollingUpdate == nil || !positiveIntOrPercent(ds.Spec.UpdateStrategy.RollingUpdate.MaxSurge) { - return false - } - return true -} - -func currentDaemonSetRevision(ctx context.Context, reader client.Reader, ds *appsv1.DaemonSet) (string, error) { - revisions := &appsv1.ControllerRevisionList{} - if err := reader.List(ctx, revisions, client.InNamespace(ds.Namespace)); err != nil { - return "", fmt.Errorf("list revisions for Agent DaemonSet %s/%s: %w", ds.Namespace, ds.Name, err) - } - var current *appsv1.ControllerRevision - for i := range revisions.Items { - revision := &revisions.Items[i] - if !controlledByUID(revision, ds.UID) { - continue - } - matches, err := controllerRevisionMatchesTemplate(revision, &ds.Spec.Template) - if err != nil { - return "", fmt.Errorf("decode revision %s for Agent DaemonSet %s/%s: %w", revision.Name, ds.Namespace, ds.Name, err) - } - if matches && (current == nil || revision.Revision > current.Revision) { - current = revision - } - } - if current == nil { - return "", nil - } - return current.Labels[appsv1.DefaultDaemonSetUniqueLabelKey], nil -} - -func controllerRevisionMatchesTemplate(revision *appsv1.ControllerRevision, template *corev1.PodTemplateSpec) (bool, error) { - var patch struct { - Spec struct { - Template corev1.PodTemplateSpec `json:"template"` - } `json:"spec"` - } - if err := json.Unmarshal(revision.Data.Raw, &patch); err != nil { - return false, err - } - return apiequality.Semantic.DeepEqual(patch.Spec.Template, *template), nil -} - -func daemonSetPods(ctx context.Context, reader client.Reader, ds *appsv1.DaemonSet) ([]corev1.Pod, error) { - selector, err := metav1.LabelSelectorAsSelector(ds.Spec.Selector) - if err != nil { - return nil, fmt.Errorf("build selector for Agent DaemonSet %s/%s: %w", ds.Namespace, ds.Name, err) - } - list := &corev1.PodList{} - if err := reader.List(ctx, list, client.InNamespace(ds.Namespace), client.MatchingLabelsSelector{Selector: selector}); err != nil { - return nil, fmt.Errorf("list Pods for Agent DaemonSet %s/%s: %w", ds.Namespace, ds.Name, err) - } - result := make([]corev1.Pod, 0, len(list.Items)) - for i := range list.Items { - if controlledByUID(&list.Items[i], ds.UID) { - result = append(result, list.Items[i]) - } - } - return result, nil -} - -func controlledByUID(obj metav1.Object, uid types.UID) bool { - owner := metav1.GetControllerOf(obj) - return owner != nil && owner.UID == uid -} - -func fallbackCandidates(ds *appsv1.DaemonSet, pods []corev1.Pod, currentRevision string, now time.Time) []fallbackCandidate { - result := make([]fallbackCandidate, 0) +func fallbackCandidates(ds *appsv1.DaemonSet, pods []corev1.Pod, revision string, now time.Time) []fallbackCandidate { + var candidates []fallbackCandidate for i := range pods { pending := &pods[i] shortage, ok := resourceOnlyUnschedulable(pending) - if !ok || !resourceFallbackSchedulingShapeSafe(pending) || pending.DeletionTimestamp != nil || pending.Spec.NodeName != "" || pending.Status.NominatedNodeName != "" || pending.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] != currentRevision { + if !ok || !resourceFallbackSchedulingShapeSafe(pending) || pending.DeletionTimestamp != nil || pending.Spec.NodeName != "" || pending.Status.NominatedNodeName != "" || pending.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] != revision { continue } nodeName, ok := targetNodeFromDaemonSetAffinity(pending) if !ok { continue } - var oldPods []*corev1.Pod for j := range pods { old := &pods[j] oldRevision := old.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] - if old.Spec.NodeName == nodeName && oldRevision != "" && oldRevision != currentRevision && podAvailable(old, ds.Spec.MinReadySeconds, now) { + if old.Spec.NodeName == nodeName && oldRevision != "" && oldRevision != revision && podAvailable(old, ds.Spec.MinReadySeconds, now) { oldPods = append(oldPods, old) } } if len(oldPods) != 1 { continue } - reservedUID := pending.Annotations[resourceFallbackOldPodAnnotation] if reservedUID != "" && reservedUID != string(oldPods[0].UID) { continue } - result = append(result, fallbackCandidate{pending: pending, old: oldPods[0], nodeName: nodeName, shortage: shortage, reserved: reservedUID != ""}) + candidates = append(candidates, fallbackCandidate{pending: pending, old: oldPods[0], nodeName: nodeName, shortage: shortage, reserved: reservedUID != ""}) } - sort.Slice(result, func(i, j int) bool { - if result[i].reserved != result[j].reserved { - return result[i].reserved + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].reserved != candidates[j].reserved { + return candidates[i].reserved } - return result[i].nodeName < result[j].nodeName + return candidates[i].nodeName < candidates[j].nodeName }) - return result + return candidates } func resourceOnlyUnschedulable(pod *corev1.Pod) (resourceShortage, bool) { @@ -415,7 +177,6 @@ func resourceOnlyUnschedulable(pod *corev1.Pod) (resourceShortage, bool) { if condition == nil || condition.Status != corev1.ConditionFalse || condition.Reason != corev1.PodReasonUnschedulable { return resourceShortage{}, false } - primary := condition.Message lower := strings.ToLower(primary) if i := strings.Index(lower, "preemption:"); i >= 0 { @@ -426,10 +187,6 @@ func resourceOnlyUnschedulable(pod *corev1.Pod) (resourceShortage, bool) { primary = primary[i+len("nodes are available:"):] } primary = strings.TrimSuffix(strings.TrimSpace(primary), ".") - if primary == "" { - return resourceShortage{}, false - } - var shortage resourceShortage for reason := range strings.SplitSeq(primary, ", ") { fields := strings.Fields(strings.ToLower(strings.TrimSpace(reason))) @@ -439,15 +196,13 @@ func resourceOnlyUnschedulable(pod *corev1.Pod) (resourceShortage, bool) { if _, err := strconv.Atoi(fields[0]); err != nil { return resourceShortage{}, false } - normalized := strings.Join(fields[1:], " ") - switch normalized { + switch strings.Join(fields[1:], " ") { case "insufficient cpu": shortage.cpu = true case "insufficient memory": shortage.memory = true case "node(s) didn't match pod's node affinity/selector", "node(s) didn't satisfy plugin(s) [nodeaffinity]": - // Expected for every non-target node because DaemonSet surge Pods are - // pinned through required node affinity. + // Expected on non-target nodes for DaemonSet surge Pods. default: return resourceShortage{}, false } @@ -474,7 +229,7 @@ func targetNodeFromDaemonSetAffinity(pod *corev1.Pod) (string, bool) { } var target string for _, term := range terms { - termTarget := "" + var termTarget string for _, requirement := range term.MatchFields { if requirement.Key != metav1.ObjectNameField { continue @@ -484,7 +239,7 @@ func targetNodeFromDaemonSetAffinity(pod *corev1.Pod) (string, bool) { } termTarget = requirement.Values[0] } - if termTarget == "" || (target != "" && termTarget != target) { + if termTarget == "" || target != "" && termTarget != target { return "", false } target = termTarget @@ -492,34 +247,31 @@ func targetNodeFromDaemonSetAffinity(pod *corev1.Pod) (string, bool) { return target, target != "" } -// resourceFallbackSchedulingShapeSafe rejects Pod-declared constraints whose -// scheduler failure could be masked by a simultaneous CPU or memory shortage. -// Cluster-specific plugins configured under the default scheduler name are not -// visible through the Pod API and must be excluded operationally. func resourceFallbackSchedulingShapeSafe(pod *corev1.Pod) bool { - if pod.Spec.SchedulerName != "" && pod.Spec.SchedulerName != corev1.DefaultSchedulerName { - return false - } - if pod.Spec.RuntimeClassName != nil || len(pod.Spec.TopologySpreadConstraints) > 0 { + if pod.Spec.SchedulerName != "" && pod.Spec.SchedulerName != corev1.DefaultSchedulerName || pod.Spec.RuntimeClassName != nil || len(pod.Spec.TopologySpreadConstraints) > 0 { return false } if pod.Spec.Affinity != nil { if pod.Spec.Affinity.PodAffinity != nil { return false } - if pod.Spec.Affinity.PodAntiAffinity != nil && !profileSurgePodAntiAffinitySafe(pod) { - return false + if pod.Spec.Affinity.PodAntiAffinity != nil { + expected, ok := profileSurgePodAntiAffinity(pod.Labels) + if !ok || !apiequality.Semantic.DeepEqual(pod.Spec.Affinity.PodAntiAffinity, expected) { + return false + } } } - for _, container := range append(append([]corev1.Container{}, pod.Spec.InitContainers...), pod.Spec.Containers...) { - for _, port := range container.Ports { + containers := append(append([]corev1.Container{}, pod.Spec.InitContainers...), pod.Spec.Containers...) + for i := range containers { + for _, port := range containers[i].Ports { if port.HostPort != 0 { return false } } } - for _, volume := range pod.Spec.Volumes { - source := volume.VolumeSource + for i := range pod.Spec.Volumes { + source := pod.Spec.Volumes[i].VolumeSource if source.EmptyDir == nil && source.HostPath == nil && source.ConfigMap == nil && source.Secret == nil && source.DownwardAPI == nil && source.Projected == nil { return false } @@ -527,32 +279,19 @@ func resourceFallbackSchedulingShapeSafe(pod *corev1.Pod) bool { return true } -// profileSurgePodAntiAffinitySafe recognizes the exact anti-affinity emitted -// for DatadogAgentProfiles. It excludes only other profiles, so deleting the -// old Pod of the same profile cannot reveal a hidden anti-affinity blocker. -func profileSurgePodAntiAffinitySafe(pod *corev1.Pod) bool { - expected, ok := profileSurgePodAntiAffinity(pod.Labels) - if !ok { - return false - } - return apiequality.Semantic.DeepEqual(pod.Spec.Affinity.PodAntiAffinity, expected) -} - func profileSurgePodAntiAffinitySatisfied(pending *corev1.Pod, nodePods []corev1.Pod) (bool, error) { - if pending.Spec.Affinity != nil && pending.Spec.Affinity.PodAntiAffinity != nil { - if !profileSurgePodAntiAffinitySafe(pending) { - return false, nil + if pending.Spec.Affinity == nil || pending.Spec.Affinity.PodAntiAffinity == nil { + return true, nil + } + for _, term := range pending.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution { + selector, err := metav1.LabelSelectorAsSelector(term.LabelSelector) + if err != nil { + return false, fmt.Errorf("parse prepared Agent Pod anti-affinity: %w", err) } - for _, term := range pending.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution { - selector, err := metav1.LabelSelectorAsSelector(term.LabelSelector) - if err != nil { - return false, fmt.Errorf("parse prepared Agent Pod anti-affinity: %w", err) - } - for i := range nodePods { - pod := &nodePods[i] - if pod.Namespace == pending.Namespace && selector.Matches(labels.Set(pod.Labels)) { - return false, nil - } + for i := range nodePods { + pod := &nodePods[i] + if pod.Namespace == pending.Namespace && selector.Matches(labels.Set(pod.Labels)) { + return false, nil } } } @@ -560,15 +299,11 @@ func profileSurgePodAntiAffinitySatisfied(pending *corev1.Pod, nodePods []corev1 } // existingPodsAllowPendingByRequiredAntiAffinity checks the symmetric half of -// inter-pod anti-affinity: an already scheduled Pod can reject the pending -// replacement even when the replacement's own terms allow it. +// inter-pod anti-affinity: a scheduled Pod can reject the pending replacement. func existingPodsAllowPendingByRequiredAntiAffinity(pending *corev1.Pod, existingPods []corev1.Pod, targetNodeName string) (bool, error) { for i := range existingPods { existing := &existingPods[i] - if existing.Spec.NodeName == "" { - continue - } - if existing.Spec.Affinity == nil || existing.Spec.Affinity.PodAntiAffinity == nil { + if existing.Spec.NodeName == "" || existing.Spec.Affinity == nil || existing.Spec.Affinity.PodAntiAffinity == nil { continue } for _, term := range existing.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution { @@ -576,15 +311,9 @@ func existingPodsAllowPendingByRequiredAntiAffinity(pending *corev1.Pod, existin if err != nil { return false, fmt.Errorf("parse existing Pod %s/%s anti-affinity: %w", existing.Namespace, existing.Name, err) } - if !selector.Matches(labels.Set(pending.Labels)) { - continue - } - if !affinityTermMaySelectNamespace(&term, existing.Namespace, pending.Namespace) { + if !selector.Matches(labels.Set(pending.Labels)) || !affinityTermMaySelectNamespace(&term, existing.Namespace, pending.Namespace) { continue } - // Pods on the target node cover hostname topology. For wider - // topologies, conservatively reject because this lightweight check - // does not fetch every Node's topology labels. if term.TopologyKey != corev1.LabelHostname || existing.Spec.NodeName == targetNodeName { return false, nil } @@ -600,18 +329,18 @@ func podAffinityTermSelector(term *corev1.PodAffinityTerm, sourceLabels map[stri } for _, key := range term.MatchLabelKeys { if value, ok := sourceLabels[key]; ok { - requirement, err := labels.NewRequirement(key, selection.In, []string{value}) - if err != nil { - return nil, err + requirement, reqErr := labels.NewRequirement(key, selection.In, []string{value}) + if reqErr != nil { + return nil, reqErr } selector = selector.Add(*requirement) } } for _, key := range term.MismatchLabelKeys { if value, ok := sourceLabels[key]; ok { - requirement, err := labels.NewRequirement(key, selection.NotIn, []string{value}) - if err != nil { - return nil, err + requirement, reqErr := labels.NewRequirement(key, selection.NotIn, []string{value}) + if reqErr != nil { + return nil, reqErr } selector = selector.Add(*requirement) } @@ -620,123 +349,49 @@ func podAffinityTermSelector(term *corev1.PodAffinityTerm, sourceLabels map[stri } func affinityTermMaySelectNamespace(term *corev1.PodAffinityTerm, sourceNamespace, targetNamespace string) bool { - if slices.Contains(term.Namespaces, targetNamespace) { - return true - } - // Namespace labels are intentionally not part of the fallback cache. Fail - // closed when a namespace selector could include the pending Pod. - if term.NamespaceSelector != nil { + if slices.Contains(term.Namespaces, targetNamespace) || term.NamespaceSelector != nil { return true } return len(term.Namespaces) == 0 && sourceNamespace == targetNamespace } -func consumedFallbackBudget(ds *appsv1.DaemonSet, pods []corev1.Pod, currentRevision string, now time.Time) int { - availableByNode := map[string]bool{} - knownNodes := map[string]bool{} - for i := range pods { - pod := &pods[i] - nodeName := pod.Spec.NodeName - if nodeName == "" { - nodeName, _ = targetNodeFromDaemonSetAffinity(pod) - } - if nodeName == "" { - continue - } - knownNodes[nodeName] = true - if podAvailable(pod, ds.Spec.MinReadySeconds, now) { - availableByNode[nodeName] = true - } - } - - reservations := 0 - reservedUnavailable := 0 - for i := range pods { - pod := &pods[i] - if pod.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] != currentRevision || pod.Annotations[resourceFallbackOldPodAnnotation] == "" || podAvailable(pod, ds.Spec.MinReadySeconds, now) { - continue - } - nodeName := pod.Spec.NodeName - if nodeName == "" { - nodeName, _ = targetNodeFromDaemonSetAffinity(pod) - } - reservations++ - if nodeName == "" || !availableByNode[nodeName] { - reservedUnavailable++ - } - } - - liveUnavailable := 0 - for nodeName := range knownNodes { - if !availableByNode[nodeName] { - liveUnavailable++ - } - } - if missingNodes := int(ds.Status.DesiredNumberScheduled) - len(knownNodes); missingNodes > 0 { - liveUnavailable += missingNodes - } - - statusUnavailable := int(ds.Status.NumberUnavailable) - statusBeyondLive := max(0, statusUnavailable-liveUnavailable) - return reservations + liveUnavailable - min(liveUnavailable, reservedUnavailable) + statusBeyondLive -} - -func podAvailable(pod *corev1.Pod, minReadySeconds int32, now time.Time) bool { - if pod.DeletionTimestamp != nil || pod.Status.Phase != corev1.PodRunning { - return false - } - for i := range pod.Status.Conditions { - condition := &pod.Status.Conditions[i] - if condition.Type != corev1.PodReady || condition.Status != corev1.ConditionTrue { - continue - } - return minReadySeconds == 0 || !condition.LastTransitionTime.IsZero() && condition.LastTransitionTime.Add(time.Duration(minReadySeconds)*time.Second).Before(now) - } - return false -} - -func (r *Reconciler) revalidateFallbackCandidate(ctx context.Context, reader client.Reader, expectedDS *appsv1.DaemonSet, candidate fallbackCandidate, expectedRevision string, requireReservation bool) (*fallbackCandidate, error) { +func (r *Reconciler) revalidateFallbackCandidate(ctx context.Context, reader client.Reader, expectedDS *appsv1.DaemonSet, candidate fallbackCandidate, revision string, requireReservation bool) (*fallbackCandidate, error) { liveDS := &appsv1.DaemonSet{} - if err := reader.Get(ctx, client.ObjectKeyFromObject(expectedDS), liveDS); err != nil { - return nil, client.IgnoreNotFound(err) + if getErr := reader.Get(ctx, client.ObjectKeyFromObject(expectedDS), liveDS); getErr != nil { + return nil, client.IgnoreNotFound(getErr) } - if liveDS.UID != expectedDS.UID || liveDS.Generation != expectedDS.Generation || !resourceFallbackDaemonSetEligible(liveDS) { + if liveDS.UID != expectedDS.UID || liveDS.Generation != expectedDS.Generation || !preparedRolloutDaemonSetEligible(liveDS) || !hasRolloutMode(liveDS.Spec.Template.Annotations) { return nil, nil } liveRevision, err := currentDaemonSetRevision(ctx, reader, liveDS) - if err != nil || liveRevision != expectedRevision { + if err != nil || liveRevision != revision { return nil, err } - pending := &corev1.Pod{} + old := &corev1.Pod{} if getErr := reader.Get(ctx, client.ObjectKeyFromObject(candidate.pending), pending); getErr != nil { return nil, client.IgnoreNotFound(getErr) } - old := &corev1.Pod{} if getErr := reader.Get(ctx, client.ObjectKeyFromObject(candidate.old), old); getErr != nil { return nil, client.IgnoreNotFound(getErr) } - if !controlledByUID(pending, liveDS.UID) || !controlledByUID(old, liveDS.UID) || pending.UID != candidate.pending.UID || old.UID != candidate.old.UID { + if pending.UID != candidate.pending.UID || old.UID != candidate.old.UID || !controlledByUID(pending, liveDS.UID) || !controlledByUID(old, liveDS.UID) { return nil, nil } shortage, ok := resourceOnlyUnschedulable(pending) nodeName, targetOK := targetNodeFromDaemonSetAffinity(pending) reservation := pending.Annotations[resourceFallbackOldPodAnnotation] - if !ok || !resourceFallbackSchedulingShapeSafe(pending) || !targetOK || nodeName != candidate.nodeName || pending.Spec.NodeName != "" || pending.Status.NominatedNodeName != "" || pending.DeletionTimestamp != nil || pending.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] != liveRevision || requireReservation && reservation != string(old.UID) || reservation != "" && reservation != string(old.UID) { + if !ok || !resourceFallbackSchedulingShapeSafe(pending) || !targetOK || nodeName != candidate.nodeName || pending.Spec.NodeName != "" || pending.Status.NominatedNodeName != "" || pending.DeletionTimestamp != nil || pending.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] != revision || requireReservation && reservation != string(old.UID) || reservation != "" && reservation != string(old.UID) { return nil, nil } - if old.Spec.NodeName != nodeName || old.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] == "" || old.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] == liveRevision || !podAvailable(old, liveDS.Spec.MinReadySeconds, time.Now()) { + if old.Spec.NodeName != nodeName || old.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] == "" || old.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] == revision || !podAvailable(old, liveDS.Spec.MinReadySeconds, time.Now()) { return nil, nil } - node := &corev1.Node{} if getErr := reader.Get(ctx, client.ObjectKey{Name: nodeName}, node); getErr != nil { return nil, client.IgnoreNotFound(getErr) } - if !nodeReadyForResourceFallback(node) { - return nil, nil - } - if !toleratesBlockingNodeTaints(pending.Spec.Tolerations, node.Spec.Taints) { + if !nodeReadyForResourceFallback(node) || !toleratesBlockingNodeTaints(pending.Spec.Tolerations, node.Spec.Taints) { return nil, nil } matches, err := nodeaffinity.GetRequiredNodeAffinity(pending).Match(node) @@ -744,34 +399,98 @@ func (r *Reconciler) revalidateFallbackCandidate(ctx context.Context, reader cli return nil, err } nodePods := &corev1.PodList{} - if listErr := reader.List(ctx, nodePods, client.MatchingFields{apiPodNodeNameField: nodeName}); listErr != nil { + if listErr := reader.List(ctx, nodePods, client.MatchingFields{"spec.nodeName": nodeName}); listErr != nil { return nil, fmt.Errorf("list Pods on node %s for Agent resource fallback: %w", nodeName, listErr) } affinitySatisfied, err := profileSurgePodAntiAffinitySatisfied(pending, nodePods.Items) - if err != nil { + if err != nil || !affinitySatisfied { return nil, err } - if !affinitySatisfied { - return nil, nil - } clusterPods := &corev1.PodList{} if listErr := reader.List(ctx, clusterPods); listErr != nil { return nil, fmt.Errorf("list cluster Pods for Agent anti-affinity fallback safety: %w", listErr) } existingAffinitySatisfied, err := existingPodsAllowPendingByRequiredAntiAffinity(pending, clusterPods.Items, nodeName) - if err != nil { + if err != nil || !existingAffinitySatisfied { return nil, err } - if !existingAffinitySatisfied { - return nil, nil - } if !resourceFitAfterOldPodRemoval(node, nodePods.Items, pending, old, shortage) { return nil, nil } - return &fallbackCandidate{pending: pending, old: old, nodeName: nodeName, shortage: shortage, reserved: reservation != ""}, nil } +func resourceFallbackBudgetWithinLimit(ctx context.Context, reader client.Reader, expectedDS *appsv1.DaemonSet, budgetValue intstr.IntOrString, revision string) (bool, error) { + liveDS := &appsv1.DaemonSet{} + if err := reader.Get(ctx, client.ObjectKeyFromObject(expectedDS), liveDS); err != nil { + return false, client.IgnoreNotFound(err) + } + if liveDS.UID != expectedDS.UID || liveDS.Generation != expectedDS.Generation || !preparedRolloutDaemonSetEligible(liveDS) { + return false, nil + } + liveRevision, err := currentDaemonSetRevision(ctx, reader, liveDS) + if err != nil || liveRevision != revision { + return false, err + } + pods, err := daemonSetPods(ctx, reader, liveDS) + if err != nil { + return false, err + } + budget, err := intstr.GetScaledValueFromIntOrPercent(&budgetValue, int(liveDS.Status.DesiredNumberScheduled), true) + if err != nil || budget <= 0 { + return false, err + } + return consumedResourceFallbackBudget(liveDS, pods, revision, time.Now()) <= budget, nil +} + +func consumedResourceFallbackBudget(ds *appsv1.DaemonSet, pods []corev1.Pod, revision string, now time.Time) int { + availableByNode := map[string]bool{} + knownNodes := map[string]bool{} + for i := range pods { + pod := &pods[i] + nodeName := pod.Spec.NodeName + if nodeName == "" { + nodeName, _ = targetNodeFromDaemonSetAffinity(pod) + } + if nodeName == "" { + continue + } + knownNodes[nodeName] = true + if podAvailable(pod, ds.Spec.MinReadySeconds, now) { + availableByNode[nodeName] = true + } + } + + reservations := 0 + reservedUnavailable := 0 + for i := range pods { + pod := &pods[i] + if pod.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] != revision || pod.Annotations[resourceFallbackOldPodAnnotation] == "" || podAvailable(pod, ds.Spec.MinReadySeconds, now) { + continue + } + nodeName := pod.Spec.NodeName + if nodeName == "" { + nodeName, _ = targetNodeFromDaemonSetAffinity(pod) + } + reservations++ + if nodeName == "" || !availableByNode[nodeName] { + reservedUnavailable++ + } + } + + liveUnavailable := 0 + for nodeName := range knownNodes { + if !availableByNode[nodeName] { + liveUnavailable++ + } + } + if missing := int(ds.Status.DesiredNumberScheduled) - len(knownNodes); missing > 0 { + liveUnavailable += missing + } + statusBeyondLive := max(0, int(ds.Status.NumberUnavailable)-liveUnavailable) + return reservations + liveUnavailable - min(liveUnavailable, reservedUnavailable) + statusBeyondLive +} + func toleratesBlockingNodeTaints(tolerations []corev1.Toleration, taints []corev1.Taint) bool { for i := range taints { taint := &taints[i] @@ -788,13 +507,8 @@ func toleratesBlockingNodeTaints(tolerations []corev1.Toleration, taints []corev if operator == "" { operator = corev1.TolerationOpEqual } - switch operator { - case corev1.TolerationOpExists: - tolerated = toleration.Key == "" || toleration.Key == taint.Key - case corev1.TolerationOpEqual: - tolerated = toleration.Key == taint.Key && toleration.Value == taint.Value - } - if tolerated { + if operator == corev1.TolerationOpExists && (toleration.Key == "" || toleration.Key == taint.Key) || operator == corev1.TolerationOpEqual && toleration.Key == taint.Key && toleration.Value == taint.Value { + tolerated = true break } } @@ -810,7 +524,7 @@ func nodeReadyForResourceFallback(node *corev1.Node) bool { return false } ready := false - pressureHealthy := map[corev1.NodeConditionType]bool{ + pressure := map[corev1.NodeConditionType]bool{ corev1.NodeMemoryPressure: false, corev1.NodeDiskPressure: false, corev1.NodePIDPressure: false, @@ -824,14 +538,14 @@ func nodeReadyForResourceFallback(node *corev1.Node) bool { if condition.Status != corev1.ConditionFalse { return false } - pressureHealthy[condition.Type] = true + pressure[condition.Type] = true case corev1.NodeNetworkUnavailable: if condition.Status != corev1.ConditionFalse { return false } } } - return ready && pressureHealthy[corev1.NodeMemoryPressure] && pressureHealthy[corev1.NodeDiskPressure] && pressureHealthy[corev1.NodePIDPressure] + return ready && pressure[corev1.NodeMemoryPressure] && pressure[corev1.NodeDiskPressure] && pressure[corev1.NodePIDPressure] } func resourceFitAfterOldPodRemoval(node *corev1.Node, nodePods []corev1.Pod, replacement, old *corev1.Pod, shortage resourceShortage) bool { @@ -840,59 +554,34 @@ func resourceFitAfterOldPodRemoval(node *corev1.Node, nodePods []corev1.Pod, rep } used := corev1.ResourceList{} oldFound := false - podCount := int64(0) for i := range nodePods { pod := &nodePods[i] if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { continue } - podCount++ addResources(used, schedulerPodRequests(pod)) - if pod.UID == old.UID { - oldFound = true - } + oldFound = oldFound || pod.UID == old.UID } if !oldFound { return false } - - replacementRequests := schedulerPodRequests(replacement) - oldRequests := schedulerPodRequests(old) before := copyResources(used) - addResources(before, replacementRequests) + addResources(before, schedulerPodRequests(replacement)) after := copyResources(used) - subtractResources(after, oldRequests) - addResources(after, replacementRequests) - - shortageStillPresent := false - for _, resourceName := range []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory} { - reported := resourceName == corev1.ResourceCPU && shortage.cpu || resourceName == corev1.ResourceMemory && shortage.memory - if !reported { - continue - } - if !resourceExceeds(before, node.Status.Allocatable, resourceName) { + subtractResources(after, schedulerPodRequests(old)) + addResources(after, schedulerPodRequests(replacement)) + for _, name := range []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory} { + reported := name == corev1.ResourceCPU && shortage.cpu || name == corev1.ResourceMemory && shortage.memory + oldRequest := schedulerPodRequests(old)[name] + if reported && (!resourceExceeds(before, node.Status.Allocatable, name) || oldRequest.Sign() <= 0) { return false } - oldRequest := oldRequests[resourceName] - if oldRequest.Sign() <= 0 { - return false - } - shortageStillPresent = true - } - if !shortageStillPresent || !resourcesFit(after, node.Status.Allocatable) { - return false } - if allocatablePods, ok := node.Status.Allocatable[corev1.ResourcePods]; ok && podCount > allocatablePods.Value() { - return false - } - return true + return resourcesFit(after, node.Status.Allocatable) } func schedulerPodRequests(pod *corev1.Pod) corev1.ResourceList { - return resourcehelper.PodRequests(pod, resourcehelper.PodResourcesOptions{ - UseStatusResources: true, - InPlacePodLevelResourcesVerticalScalingEnabled: true, - }) + return resourcehelper.PodRequests(pod, resourcehelper.PodResourcesOptions{UseStatusResources: true, InPlacePodLevelResourcesVerticalScalingEnabled: true}) } func copyResources(resources corev1.ResourceList) corev1.ResourceList { diff --git a/internal/controller/datadogagentinternal/resource_fallback_test.go b/internal/controller/datadogagentinternal/resource_fallback_test.go index 12928b4fb8..3bceee938f 100644 --- a/internal/controller/datadogagentinternal/resource_fallback_test.go +++ b/internal/controller/datadogagentinternal/resource_fallback_test.go @@ -7,8 +7,6 @@ package datadogagentinternal import ( "context" "encoding/json" - "errors" - "maps" "testing" "time" @@ -17,934 +15,115 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - datadoghqcommon "github.com/DataDog/datadog-operator/api/datadoghq/common" datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" - datadoghqv2alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v2alpha1" - componentagent "github.com/DataDog/datadog-operator/internal/controller/datadogagent/component/agent" - "github.com/DataDog/datadog-operator/pkg/constants" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestConfigureResourceFallback(t *testing.T) { - tests := []struct { - name string - strategyType appsv1.DaemonSetUpdateStrategyType - maxSurge *intstr.IntOrString - budget intstr.IntOrString - enabled bool - wantMaxSurge *intstr.IntOrString - wantUnavailable *intstr.IntOrString - }{ - { - name: "surge is bounded by the existing percentage budget", - maxSurge: ptr.To(intstr.FromString("100%")), - budget: intstr.FromString("20%"), - enabled: true, - wantMaxSurge: ptr.To(intstr.FromString("20%")), - wantUnavailable: ptr.To(intstr.FromInt(0)), - }, - { - name: "absolute budget", - maxSurge: ptr.To(intstr.FromInt(20)), - budget: intstr.FromInt(2), - enabled: true, - wantMaxSurge: ptr.To(intstr.FromInt(2)), - wantUnavailable: ptr.To(intstr.FromInt(0)), - }, - { - name: "surge remains opt in", - budget: intstr.FromInt(1), - wantMaxSurge: nil, - }, - { - name: "on delete is untouched", - strategyType: appsv1.OnDeleteDaemonSetStrategyType, - maxSurge: ptr.To(intstr.FromInt(1)), - budget: intstr.FromInt(1), - wantMaxSurge: ptr.To(intstr.FromInt(1)), - }, - { - name: "zero budget disables fallback but preserves requested surge", - maxSurge: ptr.To(intstr.FromInt(3)), - budget: intstr.FromInt(0), - wantMaxSurge: ptr.To(intstr.FromInt(3)), - wantUnavailable: ptr.To(intstr.FromInt(0)), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ds := &appsv1.DaemonSet{Spec: appsv1.DaemonSetSpec{UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ - Type: tt.strategyType, - RollingUpdate: &appsv1.RollingUpdateDaemonSet{MaxSurge: tt.maxSurge}, - }}} - assert.Equal(t, tt.enabled, configureResourceFallback(ds, tt.budget)) - assert.Equal(t, tt.wantMaxSurge, ds.Spec.UpdateStrategy.RollingUpdate.MaxSurge) - assert.Equal(t, tt.wantUnavailable, ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable) - }) - } -} - -func TestResourceFallbackBudgetPrecedence(t *testing.T) { - overrideBudget := intstr.FromString("25%") - ddai := &datadoghqv1alpha1.DatadogAgentInternal{Spec: datadoghqv2alpha1.DatadogAgentSpec{ - Override: map[datadoghqv2alpha1.ComponentName]*datadoghqv2alpha1.DatadogAgentComponentOverride{ - datadoghqv2alpha1.NodeAgentComponentName: { - UpdateStrategy: &datadoghqcommon.UpdateStrategy{RollingUpdate: &datadoghqcommon.RollingUpdate{MaxUnavailable: &overrideBudget}}, - }, - }, - }} - options := &componentagent.ExtendedDaemonsetOptions{MaxPodUnavailable: "2"} - - assert.Equal(t, overrideBudget, resourceFallbackBudget(ddai, options), "the DatadogAgent override is the requested rollout budget") - ddai.Spec.Override = nil - assert.Equal(t, intstr.FromInt(2), resourceFallbackBudget(ddai, options), "the Operator option is the compatibility fallback") - assert.Equal(t, intstr.FromInt(defaultFallbackMaxUnavailable), resourceFallbackBudget(ddai, nil), "the fallback remains bounded when neither source is configured") -} - -func TestResourceOnlyUnschedulable(t *testing.T) { - tests := []struct { - name string - reason string - message string - want resourceShortage - ok bool - }{ - {name: "cpu", reason: corev1.PodReasonUnschedulable, message: "0/1 nodes are available: 1 Insufficient cpu.", want: resourceShortage{cpu: true}, ok: true}, - {name: "memory and pinning affinity", reason: corev1.PodReasonUnschedulable, message: "0/3 nodes are available: 1 Insufficient memory, 2 node(s) didn't satisfy plugin(s) [NodeAffinity]. preemption: 0/3 nodes are available: 3 Preemption is not helpful for scheduling.", want: resourceShortage{memory: true}, ok: true}, - {name: "cpu and memory", reason: corev1.PodReasonUnschedulable, message: "0/1 nodes are available: 1 Insufficient cpu, 1 Insufficient memory.", want: resourceShortage{cpu: true, memory: true}, ok: true}, - {name: "taint is rejected", reason: corev1.PodReasonUnschedulable, message: "0/1 nodes are available: 1 Insufficient cpu, 1 node(s) had untolerated taint.", ok: false}, - {name: "host port is rejected", reason: corev1.PodReasonUnschedulable, message: "0/1 nodes are available: 1 Insufficient cpu, 1 node(s) didn't have free ports for the requested pod ports.", ok: false}, - {name: "ephemeral storage is rejected", reason: corev1.PodReasonUnschedulable, message: "0/1 nodes are available: 1 Insufficient ephemeral-storage.", ok: false}, - {name: "custom reason containing cpu text is rejected", reason: corev1.PodReasonUnschedulable, message: "0/1 nodes are available: 1 custom plugin: Insufficient cpu.", ok: false}, - {name: "wrong condition reason", reason: "SchedulingGated", message: "0/1 nodes are available: 1 Insufficient cpu.", ok: false}, - {name: "empty primary reason", reason: corev1.PodReasonUnschedulable, message: "preemption:", ok: false}, - {name: "reason without count", reason: corev1.PodReasonUnschedulable, message: "0/1 nodes are available: cpu.", ok: false}, - {name: "non-numeric count", reason: corev1.PodReasonUnschedulable, message: "0/1 nodes are available: many Insufficient cpu.", ok: false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - pod := &corev1.Pod{Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: tt.reason, Message: tt.message}}}} - got, ok := resourceOnlyUnschedulable(pod) - assert.Equal(t, tt.ok, ok) - assert.Equal(t, tt.want, got) - }) - } -} - -func TestTargetNodeFromDaemonSetAffinity(t *testing.T) { - requirement := func(operator corev1.NodeSelectorOperator, values ...string) corev1.NodeSelectorRequirement { - return corev1.NodeSelectorRequirement{Key: metav1.ObjectNameField, Operator: operator, Values: values} - } - podWithTerms := func(terms ...corev1.NodeSelectorTerm) *corev1.Pod { - return &corev1.Pod{Spec: corev1.PodSpec{Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: terms}, - }}}} - } - - tests := []struct { - name string - pod *corev1.Pod - want string - ok bool - }{ - {name: "daemonset target", pod: pendingPodForNode("node-a"), want: "node-a", ok: true}, - {name: "no affinity", pod: &corev1.Pod{}}, - {name: "no node affinity", pod: &corev1.Pod{Spec: corev1.PodSpec{Affinity: &corev1.Affinity{}}}}, - {name: "no required node affinity", pod: &corev1.Pod{Spec: corev1.PodSpec{Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{}}}}}, - {name: "empty terms", pod: podWithTerms()}, - {name: "term without target field", pod: podWithTerms(corev1.NodeSelectorTerm{MatchFields: []corev1.NodeSelectorRequirement{{Key: "metadata.namespace", Operator: corev1.NodeSelectorOpIn, Values: []string{"datadog"}}}})}, - {name: "wrong operator", pod: podWithTerms(corev1.NodeSelectorTerm{MatchFields: []corev1.NodeSelectorRequirement{requirement(corev1.NodeSelectorOpNotIn, "node-a")}})}, - {name: "multiple values", pod: podWithTerms(corev1.NodeSelectorTerm{MatchFields: []corev1.NodeSelectorRequirement{requirement(corev1.NodeSelectorOpIn, "node-a", "node-b")}})}, - {name: "duplicate target field", pod: podWithTerms(corev1.NodeSelectorTerm{MatchFields: []corev1.NodeSelectorRequirement{requirement(corev1.NodeSelectorOpIn, "node-a"), requirement(corev1.NodeSelectorOpIn, "node-a")}})}, - {name: "consistent terms", pod: podWithTerms( - corev1.NodeSelectorTerm{MatchFields: []corev1.NodeSelectorRequirement{requirement(corev1.NodeSelectorOpIn, "node-a")}}, - corev1.NodeSelectorTerm{MatchFields: []corev1.NodeSelectorRequirement{requirement(corev1.NodeSelectorOpIn, "node-a")}}, - ), want: "node-a", ok: true}, - {name: "conflicting terms", pod: podWithTerms( - corev1.NodeSelectorTerm{MatchFields: []corev1.NodeSelectorRequirement{requirement(corev1.NodeSelectorOpIn, "node-a")}}, - corev1.NodeSelectorTerm{MatchFields: []corev1.NodeSelectorRequirement{requirement(corev1.NodeSelectorOpIn, "node-b")}}, - )}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, ok := targetNodeFromDaemonSetAffinity(tt.pod) - assert.Equal(t, tt.ok, ok) - assert.Equal(t, tt.want, got) - }) - } -} - -func TestPrepareProfileAntiAffinityForSurge(t *testing.T) { - labels := map[string]string{ - datadoghqcommon.AgentDeploymentNameLabelKey: "datadog-agent", - constants.ProfileLabelKey: "linux", - } - - t.Run("no anti-affinity", func(t *testing.T) { - for _, template := range []*corev1.PodTemplateSpec{ - {}, - {Spec: corev1.PodSpec{Affinity: &corev1.Affinity{}}}, - } { - assert.True(t, prepareProfileAntiAffinityForSurge(template)) - } - }) - - t.Run("custom anti-affinity is rejected without mutation", func(t *testing.T) { - template := &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Affinity: &corev1.Affinity{PodAntiAffinity: &corev1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{TopologyKey: "topology.kubernetes.io/zone"}}, - }}}} - original := template.DeepCopy() - assert.False(t, prepareProfileAntiAffinityForSurge(template)) - assert.Equal(t, original, template) - }) - - t.Run("missing deployment identity is rejected", func(t *testing.T) { - template := &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Affinity: &corev1.Affinity{PodAntiAffinity: broadAgentPodAntiAffinity()}}} - assert.False(t, prepareProfileAntiAffinityForSurge(template)) - }) - - t.Run("standard affinity is narrowed", func(t *testing.T) { - template := &corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: corev1.PodSpec{ - Affinity: &corev1.Affinity{PodAntiAffinity: broadAgentPodAntiAffinity()}, - }} - expected, ok := profileSurgePodAntiAffinity(labels) - require.True(t, ok) - require.True(t, prepareProfileAntiAffinityForSurge(template)) - assert.Equal(t, expected, template.Spec.Affinity.PodAntiAffinity) - }) -} - -func TestResourceFallbackSchedulingShapeAllowsOnlyProfileSurgeAntiAffinity(t *testing.T) { - namedLabels := map[string]string{ - datadoghqcommon.AgentDeploymentNameLabelKey: "datadog-agent", - constants.ProfileLabelKey: "linux", - } - namedAntiAffinity, ok := profileSurgePodAntiAffinity(namedLabels) - require.True(t, ok) - named := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Labels: namedLabels}, Spec: corev1.PodSpec{ - Affinity: &corev1.Affinity{PodAntiAffinity: namedAntiAffinity}, - Containers: []corev1.Container{{Name: "agent"}}, - }} - assert.True(t, resourceFallbackSchedulingShapeSafe(named)) - - defaultProfile := named.DeepCopy() - defaultProfile.Labels = map[string]string{datadoghqcommon.AgentDeploymentNameLabelKey: "datadog-agent"} - defaultProfile.Spec.Affinity.PodAntiAffinity, ok = profileSurgePodAntiAffinity(defaultProfile.Labels) - require.True(t, ok) - assert.True(t, resourceFallbackSchedulingShapeSafe(defaultProfile)) - - wrongProfile := named.DeepCopy() - wrongProfile.Spec.Affinity.PodAntiAffinity, ok = profileSurgePodAntiAffinity(map[string]string{ - datadoghqcommon.AgentDeploymentNameLabelKey: "datadog-agent", - constants.ProfileLabelKey: "gpu", - }) - require.True(t, ok) - assert.False(t, resourceFallbackSchedulingShapeSafe(wrongProfile)) - - custom := named.DeepCopy() - custom.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].TopologyKey = "topology.kubernetes.io/zone" - assert.False(t, resourceFallbackSchedulingShapeSafe(custom)) -} - -func TestProfileSurgePodAntiAffinityIdentity(t *testing.T) { - incomingLabels := map[string]string{ - datadoghqcommon.AgentDeploymentNameLabelKey: "datadog-agent", - constants.ProfileLabelKey: "linux", - } - antiAffinity, ok := profileSurgePodAntiAffinity(incomingLabels) - require.True(t, ok) - - conflicts := func(existingLabels map[string]string) bool { - for _, term := range antiAffinity.RequiredDuringSchedulingIgnoredDuringExecution { - selector, err := metav1.LabelSelectorAsSelector(term.LabelSelector) - require.NoError(t, err) - if selector.Matches(labels.Set(existingLabels)) { - return true - } - } - return false - } - - assert.False(t, conflicts(map[string]string{ - datadoghqcommon.AgentDeploymentComponentLabelKey: constants.DefaultAgentResourceSuffix, - datadoghqcommon.AgentDeploymentNameLabelKey: "datadog-agent", - constants.ProfileLabelKey: "linux", - }), "old and new revisions of the same DDA profile may overlap") - assert.True(t, conflicts(map[string]string{ - datadoghqcommon.AgentDeploymentComponentLabelKey: constants.DefaultAgentResourceSuffix, - datadoghqcommon.AgentDeploymentNameLabelKey: "datadog-agent", - constants.ProfileLabelKey: "gpu", - }), "another profile of the same DDA must remain excluded") - assert.True(t, conflicts(map[string]string{ - datadoghqcommon.AgentDeploymentComponentLabelKey: constants.DefaultAgentResourceSuffix, - datadoghqcommon.AgentDeploymentNameLabelKey: "other-datadog-agent", - constants.ProfileLabelKey: "linux", - }), "the same profile name from another DDA must remain excluded") -} - -func TestProfileSurgePodAntiAffinitySatisfiedOnTargetNode(t *testing.T) { - pendingLabels := map[string]string{ - datadoghqcommon.AgentDeploymentNameLabelKey: "datadog-agent", - datadoghqcommon.AgentDeploymentComponentLabelKey: constants.DefaultAgentResourceSuffix, - constants.ProfileLabelKey: "linux", - } - antiAffinity, ok := profileSurgePodAntiAffinity(pendingLabels) - require.True(t, ok) - pending := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "datadog", Labels: pendingLabels}, Spec: corev1.PodSpec{ - Affinity: &corev1.Affinity{PodAntiAffinity: antiAffinity}, - }} - sameProfile := corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "datadog", Labels: maps.Clone(pendingLabels)}} - - satisfied, err := profileSurgePodAntiAffinitySatisfied(pending, []corev1.Pod{sameProfile}) - require.NoError(t, err) - assert.True(t, satisfied) - - otherProfile := sameProfile.DeepCopy() - otherProfile.Labels[constants.ProfileLabelKey] = "gpu" - satisfied, err = profileSurgePodAntiAffinitySatisfied(pending, []corev1.Pod{sameProfile, *otherProfile}) - require.NoError(t, err) - assert.False(t, satisfied, "a masked different-profile blocker must prevent old Pod deletion") - - otherProfile.Namespace = "another-namespace" - satisfied, err = profileSurgePodAntiAffinitySatisfied(pending, []corev1.Pod{sameProfile, *otherProfile}) - require.NoError(t, err) - assert.True(t, satisfied, "Pod anti-affinity without explicit namespaces is namespace-scoped") -} - -func TestExistingPodRequiredAntiAffinityCanRejectReplacement(t *testing.T) { - pending := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "datadog", Labels: map[string]string{ - datadoghqcommon.AgentDeploymentComponentLabelKey: constants.DefaultAgentResourceSuffix, - "rollout": "new", - }}} - existing := corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "datadog", Name: "peer", Labels: map[string]string{"rollout": "old"}}, Spec: corev1.PodSpec{ - NodeName: "node-a", - Affinity: &corev1.Affinity{PodAntiAffinity: &corev1.PodAntiAffinity{RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ - datadoghqcommon.AgentDeploymentComponentLabelKey: constants.DefaultAgentResourceSuffix, - }}, - TopologyKey: corev1.LabelHostname, - }}}}, - }} - - allowed, err := existingPodsAllowPendingByRequiredAntiAffinity(pending, []corev1.Pod{existing}, "node-a") - require.NoError(t, err) - assert.False(t, allowed, "an existing Pod's required anti-affinity must block fallback deletion") - - existing.Namespace = "another-namespace" - allowed, err = existingPodsAllowPendingByRequiredAntiAffinity(pending, []corev1.Pod{existing}, "node-a") - require.NoError(t, err) - assert.True(t, allowed) - - emptyNamespaceSelector := &metav1.LabelSelector{} - existing.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].NamespaceSelector = emptyNamespaceSelector - allowed, err = existingPodsAllowPendingByRequiredAntiAffinity(pending, []corev1.Pod{existing}, "node-a") - require.NoError(t, err) - assert.False(t, allowed, "namespace selectors fail closed because namespace labels are not cached") - - existing.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector = &metav1.LabelSelector{MatchLabels: map[string]string{"rollout": "old"}} - existing.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].NamespaceSelector = nil - existing.Namespace = "datadog" - allowed, err = existingPodsAllowPendingByRequiredAntiAffinity(pending, []corev1.Pod{existing}, "node-a") - require.NoError(t, err) - assert.True(t, allowed, "non-matching selectors do not block the replacement") - - existing.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector = &metav1.LabelSelector{MatchLabels: map[string]string{ - datadoghqcommon.AgentDeploymentComponentLabelKey: constants.DefaultAgentResourceSuffix, - }} - existing.Spec.NodeName = "node-b" - allowed, err = existingPodsAllowPendingByRequiredAntiAffinity(pending, []corev1.Pod{existing}, "node-a") - require.NoError(t, err) - assert.True(t, allowed, "hostname anti-affinity on another node does not block") - - existing.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].TopologyKey = "topology.kubernetes.io/zone" - allowed, err = existingPodsAllowPendingByRequiredAntiAffinity(pending, []corev1.Pod{existing}, "node-a") - require.NoError(t, err) - assert.False(t, allowed, "wider topology terms fail closed without loading every Node's topology labels") -} - -func TestAffinityTermMaySelectNamespace(t *testing.T) { - assert.True(t, affinityTermMaySelectNamespace(&corev1.PodAffinityTerm{Namespaces: []string{"target"}}, "source", "target")) - assert.False(t, affinityTermMaySelectNamespace(&corev1.PodAffinityTerm{Namespaces: []string{"other"}}, "source", "target")) -} - -func TestPodAffinityTermSelector(t *testing.T) { - term := &corev1.PodAffinityTerm{ - LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "agent"}}, - MatchLabelKeys: []string{"rollout"}, - MismatchLabelKeys: []string{"profile"}, - } - selector, err := podAffinityTermSelector(term, map[string]string{"rollout": "new", "profile": "linux"}) - require.NoError(t, err) - assert.True(t, selector.Matches(labels.Set{"app": "agent", "rollout": "new", "profile": "gpu"})) - assert.False(t, selector.Matches(labels.Set{"app": "agent", "rollout": "old", "profile": "gpu"})) - assert.False(t, selector.Matches(labels.Set{"app": "agent", "rollout": "new", "profile": "linux"})) - - selector, err = podAffinityTermSelector(term, nil) - require.NoError(t, err) - assert.True(t, selector.Matches(labels.Set{"app": "agent"}), "keys missing from the source Pod must not add selector requirements") - - _, err = podAffinityTermSelector(&corev1.PodAffinityTerm{LabelSelector: &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{{ - Key: "app", Operator: metav1.LabelSelectorOperator("Invalid"), Values: []string{"agent"}, - }}}}, nil) - require.Error(t, err) - - _, err = podAffinityTermSelector(&corev1.PodAffinityTerm{ - LabelSelector: &metav1.LabelSelector{}, - MatchLabelKeys: []string{"bad key"}, - }, map[string]string{"bad key": "value"}) - require.Error(t, err) - - _, err = podAffinityTermSelector(&corev1.PodAffinityTerm{ - LabelSelector: &metav1.LabelSelector{}, - MismatchLabelKeys: []string{"bad key"}, - }, map[string]string{"bad key": "value"}) - require.Error(t, err) -} - -func TestResourceFitAfterOldPodRemoval(t *testing.T) { - node := &corev1.Node{Status: corev1.NodeStatus{Allocatable: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1500m"), - corev1.ResourceMemory: resource.MustParse("1Gi"), - corev1.ResourcePods: resource.MustParse("10"), - }}} - old := scheduledResourcePod("old", "old-uid", "node-a", "1", "128Mi") - replacement := scheduledResourcePod("new", "new-uid", "", "1", "128Mi") - - assert.True(t, resourceFitAfterOldPodRemoval(node, []corev1.Pod{*old}, replacement, old, resourceShortage{cpu: true})) - - tooLarge := replacement.DeepCopy() - tooLarge.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = resource.MustParse("2") - assert.False(t, resourceFitAfterOldPodRemoval(node, []corev1.Pod{*old}, tooLarge, old, resourceShortage{cpu: true}), "replacement must fit after old exits") - - noCPUOld := old.DeepCopy() - noCPUOld.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = resource.MustParse("0") - assert.False(t, resourceFitAfterOldPodRemoval(node, []corev1.Pod{*noCPUOld}, replacement, noCPUOld, resourceShortage{cpu: true}), "old Pod must contribute to the shortage") - - staleMessage := node.DeepCopy() - staleMessage.Status.Allocatable[corev1.ResourceCPU] = resource.MustParse("3") - assert.False(t, resourceFitAfterOldPodRemoval(staleMessage, []corev1.Pod{*old}, replacement, old, resourceShortage{cpu: true}), "reported shortage must still be observable") - - claims := replacement.DeepCopy() - claims.Spec.ResourceClaims = []corev1.PodResourceClaim{{Name: "accelerator"}} - assert.False(t, resourceFitAfterOldPodRemoval(node, []corev1.Pod{*old}, claims, old, resourceShortage{cpu: true}), "dynamic resource claims are not modeled") - - assert.False(t, resourceFitAfterOldPodRemoval(node, nil, replacement, old, resourceShortage{cpu: true}), "the exact old Pod must still be present") - assert.False(t, resourceFitAfterOldPodRemoval(node, []corev1.Pod{*old}, replacement, old, resourceShortage{}), "a scheduler-reported CPU or memory shortage is required") - - podLimited := node.DeepCopy() - podLimited.Status.Allocatable[corev1.ResourcePods] = resource.MustParse("0") - assert.False(t, resourceFitAfterOldPodRemoval(podLimited, []corev1.Pod{*old}, replacement, old, resourceShortage{cpu: true}), "the replacement must fit the node Pod limit") - - finished := scheduledResourcePod("finished", "finished-uid", "node-a", "10", "10Gi") - finished.Status.Phase = corev1.PodSucceeded - assert.True(t, resourceFitAfterOldPodRemoval(node, []corev1.Pod{*old, *finished}, replacement, old, resourceShortage{cpu: true}), "terminal Pods do not consume scheduler capacity") -} - -func TestSchedulerPodRequestsIncludesInitAndOverhead(t *testing.T) { - pod := scheduledResourcePod("pod", "uid", "node-a", "250m", "100Mi") - pod.Spec.InitContainers = []corev1.Container{{Name: "init", Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}}}} - pod.Spec.Overhead = corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")} - requests := schedulerPodRequests(pod) - assert.Equal(t, int64(1100), requests.Cpu().MilliValue()) -} - -func TestConsumedFallbackBudget(t *testing.T) { - now := time.Now() - ds := &appsv1.DaemonSet{Spec: appsv1.DaemonSetSpec{MinReadySeconds: 0}, Status: appsv1.DaemonSetStatus{NumberUnavailable: 1}} - old := readyPod("old", "old-uid", "node-a", "old", now.Add(-time.Minute)) - reserved := pendingPodForNode("node-a") - reserved.Name = "new" - reserved.Labels = map[string]string{appsv1.DefaultDaemonSetUniqueLabelKey: "new"} - reserved.Annotations = map[string]string{resourceFallbackOldPodAnnotation: "old-uid"} - - assert.Equal(t, 2, consumedFallbackBudget(ds, []corev1.Pod{*old, *reserved}, "new", now), "reservation is separate while old remains available") - old.DeletionTimestamp = &metav1.Time{Time: now} - assert.Equal(t, 1, consumedFallbackBudget(ds, []corev1.Pod{*old, *reserved}, "new", now), "reservation overlaps status once its node is unavailable") -} - -func TestReconcileResourceFallbackDeletesOnlyResourceBlockingOldPod(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) +func TestResourceFallbackDeletesOnlyOldPodThatMakesReplacementFit(t *testing.T) { + fixture := newResourceFallbackFixture(t) result, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) require.NoError(t, err) assert.Equal(t, time.Second, result.RequeueAfter) err = fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{}) - assert.True(t, apierrors.IsNotFound(err), "old Pod should be deleted") - updatedPending := &corev1.Pod{} - require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.pending), updatedPending)) - assert.Equal(t, string(fixture.old.UID), updatedPending.Annotations[resourceFallbackOldPodAnnotation]) + assert.True(t, apierrors.IsNotFound(err)) + require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.pending), fixture.pending)) + assert.Equal(t, string(fixture.old.UID), fixture.pending.Annotations[resourceFallbackOldPodAnnotation]) } -func TestReconcileResourceFallbackKeepsOldPodDuringNodePressure(t *testing.T) { - conditions := healthyNodeConditions() - for i := range conditions { - if conditions[i].Type == corev1.NodeDiskPressure { - conditions[i].Status = corev1.ConditionTrue - } - } - fixture := newFallbackTestFixture(t, conditions) - result, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) - require.NoError(t, err) - assert.Zero(t, result.RequeueAfter, "permanently ineligible candidates must not cause a one-second polling loop") - require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{}), "old Pod must remain during DiskPressure") - updatedPending := &corev1.Pod{} - require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.pending), updatedPending)) - assert.Empty(t, updatedPending.Annotations[resourceFallbackOldPodAnnotation], "ineligible fallback must not reserve budget") -} - -func TestReconcileResourceFallbackKeepsOldPodForHiddenSchedulerConstraints(t *testing.T) { - tests := []struct { +func TestResourceFallbackFailsClosedForOtherSchedulingBlockers(t *testing.T) { + for _, tt := range []struct { name string - mutate func(*corev1.Pod) + mutate func(*resourceFallbackFixture) }{ - { - name: "persistent volume", - mutate: func(pod *corev1.Pod) { - pod.Spec.Volumes = []corev1.Volume{{Name: "data", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"}}}} - }, - }, - { - name: "pod affinity", - mutate: func(pod *corev1.Pod) { - pod.Spec.Affinity.PodAffinity = &corev1.PodAffinity{RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{TopologyKey: "kubernetes.io/hostname", LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "peer"}}}}} - }, - }, - { - name: "unrecognized pod anti affinity", - mutate: func(pod *corev1.Pod) { - pod.Spec.Affinity.PodAntiAffinity = &corev1.PodAntiAffinity{RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{TopologyKey: "topology.kubernetes.io/zone"}}} - }, - }, - { - name: "declared host port", - mutate: func(pod *corev1.Pod) { - pod.Spec.Containers[0].Ports = []corev1.ContainerPort{{ContainerPort: 8126, HostPort: 8126}} - }, - }, - { - name: "topology spread", - mutate: func(pod *corev1.Pod) { - pod.Spec.TopologySpreadConstraints = []corev1.TopologySpreadConstraint{{MaxSkew: 1, TopologyKey: "kubernetes.io/hostname", WhenUnsatisfiable: corev1.DoNotSchedule, LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "agent"}}}} - }, - }, - { - name: "custom scheduler", - mutate: func(pod *corev1.Pod) { - pod.Spec.SchedulerName = "custom-scheduler" - }, - }, - } - - for _, tt := range tests { + {name: "mixed scheduler reason", mutate: func(f *resourceFallbackFixture) { + f.pending.Status.Conditions[0].Message = "0/1 nodes are available: 1 node(s) didn't have free ports for the requested pod ports, 1 Insufficient cpu." + }}, + {name: "hostPort introduced by admission", mutate: func(f *resourceFallbackFixture) { + f.pending.Spec.Containers[0].Ports = []corev1.ContainerPort{{ContainerPort: 8126, HostPort: 8126}} + }}, + {name: "old requests are insufficient", mutate: func(f *resourceFallbackFixture) { + f.pending.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = resource.MustParse("2") + }}, + } { t.Run(tt.name, func(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - pending := &corev1.Pod{} - require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.pending), pending)) - tt.mutate(pending) - require.NoError(t, fixture.client.Update(context.Background(), pending)) - _, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) - require.NoError(t, err) - require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{}), "old Pod must remain") - updatedPending := &corev1.Pod{} - require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.pending), updatedPending)) - assert.Empty(t, updatedPending.Annotations[resourceFallbackOldPodAnnotation]) - }) - } -} - -func TestReconcileResourceFallbackUsesLiveUnavailablePodsWhenStatusLags(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - liveDS := &appsv1.DaemonSet{} - require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.ds), liveDS)) - liveDS.Status.DesiredNumberScheduled = 2 - require.NoError(t, fixture.client.Status().Update(context.Background(), liveDS)) - - unavailable := readyPod("unavailable", "unavailable-uid", "node-b", "old-revision", time.Now().Add(-time.Minute)) - unavailable.Namespace = fixture.ds.Namespace - unavailable.Labels["app"] = "agent" - unavailable.OwnerReferences = []metav1.OwnerReference{daemonSetOwner(fixture.ds)} - unavailable.Status.Conditions[0].Status = corev1.ConditionFalse - require.NoError(t, fixture.client.Create(context.Background(), unavailable)) - - result, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) - require.NoError(t, err) - assert.Zero(t, result.RequeueAfter, "an already-consumed budget must not cause a one-second polling loop") - require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{}), "fallback must not exceed maxUnavailable while DaemonSet status lags") -} - -func TestReconcileResourceFallbackRejectsForeignDaemonSet(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - liveDS := &appsv1.DaemonSet{} - require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.ds), liveDS)) - liveDS.OwnerReferences[0].UID = "foreign-ddai" - require.NoError(t, fixture.client.Update(context.Background(), liveDS)) - - _, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) - require.NoError(t, err) - require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{}), "foreign DaemonSet Pods must never be deleted") -} - -func TestReconcileResourceFallbackEarlyExitsAndErrors(t *testing.T) { - t.Run("uses cached client when API reader is absent", func(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - fixture.reconciler.apiReader = nil - result, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(0)) - require.NoError(t, err) - assert.Zero(t, result) - }) - - t.Run("missing daemonset", func(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - require.NoError(t, fixture.client.Delete(context.Background(), fixture.ds)) - result, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) - require.NoError(t, err) - assert.Zero(t, result) - }) - - t.Run("daemonset read error", func(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - reader := interceptor.NewClient(fixture.client.(client.WithWatch), interceptor.Funcs{ - Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { - if _, ok := obj.(*appsv1.DaemonSet); ok { - return errors.New("read daemonset") - } - return c.Get(ctx, key, obj, opts...) - }, - }) - fixture.reconciler.apiReader = reader - _, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) - require.ErrorContains(t, err, "get Agent DaemonSet") - }) - - t.Run("invalid and zero budgets", func(t *testing.T) { - for _, budget := range []intstr.IntOrString{intstr.FromString("invalid"), intstr.FromInt(0)} { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - result, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, budget) - if budget.Type == intstr.String { - require.ErrorContains(t, err, "resolve Agent resource fallback budget") - } else { - require.NoError(t, err) - assert.Zero(t, result) + fixture := newResourceFallbackFixture(t) + tt.mutate(&fixture) + if tt.name == "mixed scheduler reason" { + _, safe := resourceOnlyUnschedulable(fixture.pending) + require.False(t, safe) } - } - }) - - t.Run("missing current revision", func(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - revisions := &appsv1.ControllerRevisionList{} - require.NoError(t, fixture.client.List(context.Background(), revisions)) - require.NotEmpty(t, revisions.Items) - require.NoError(t, fixture.client.Delete(context.Background(), &revisions.Items[0])) - result, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) - require.NoError(t, err) - assert.Zero(t, result) - }) - - t.Run("revision and pod list errors", func(t *testing.T) { - for _, failPods := range []bool{false, true} { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - reader := interceptor.NewClient(fixture.client.(client.WithWatch), interceptor.Funcs{ - List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { - if _, ok := list.(*appsv1.ControllerRevisionList); ok && !failPods { - return errors.New("list revisions") - } - if _, ok := list.(*corev1.PodList); ok && failPods { - return errors.New("list pods") - } - return c.List(ctx, list, opts...) - }, - }) - fixture.reconciler.apiReader = reader + desiredStatus := *fixture.pending.Status.DeepCopy() + require.NoError(t, fixture.client.Update(context.Background(), fixture.pending)) + livePending := &corev1.Pod{} + require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.pending), livePending)) + livePending.Status = desiredStatus + require.NoError(t, fixture.client.Status().Update(context.Background(), livePending)) + fixture.pending = livePending _, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) - require.Error(t, err) - } - }) - - t.Run("reservation patch error", func(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - writer := interceptor.NewClient(fixture.client.(client.WithWatch), interceptor.Funcs{ - Patch: func(context.Context, client.WithWatch, client.Object, client.Patch, ...client.PatchOption) error { - return errors.New("patch reservation") - }, - }) - fixture.reconciler.client = writer - _, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) - require.ErrorContains(t, err, "reserve Agent resource fallback") - }) - - t.Run("old pod delete error", func(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - writer := interceptor.NewClient(fixture.client.(client.WithWatch), interceptor.Funcs{ - Delete: func(context.Context, client.WithWatch, client.Object, ...client.DeleteOption) error { - return errors.New("delete old pod") - }, - }) - fixture.reconciler.client = writer - _, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) - require.ErrorContains(t, err, "delete old Agent Pod") - }) -} - -func TestFallbackBudgetWithinLimitFailsClosed(t *testing.T) { - t.Run("missing DaemonSet", func(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - require.NoError(t, fixture.client.Delete(context.Background(), fixture.ds)) - ok, err := fallbackBudgetWithinLimit(context.Background(), fixture.client, fixture.ds, intstr.FromInt(1), "new-revision") - require.NoError(t, err) - assert.False(t, ok) - }) - - t.Run("stale DaemonSet identity", func(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - expected := fixture.ds.DeepCopy() - expected.UID = "stale-uid" - ok, err := fallbackBudgetWithinLimit(context.Background(), fixture.client, expected, intstr.FromInt(1), "new-revision") - require.NoError(t, err) - assert.False(t, ok) - }) - - t.Run("revision changed", func(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - ok, err := fallbackBudgetWithinLimit(context.Background(), fixture.client, fixture.ds, intstr.FromInt(1), "other-revision") - require.NoError(t, err) - assert.False(t, ok) - }) - - t.Run("invalid budget", func(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - ok, err := fallbackBudgetWithinLimit(context.Background(), fixture.client, fixture.ds, intstr.FromString("invalid"), "new-revision") - require.Error(t, err) - assert.False(t, ok) - }) -} - -func TestControllerRevisionMatchesTemplateRejectsInvalidData(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - revisions := &appsv1.ControllerRevisionList{} - require.NoError(t, fixture.client.List(context.Background(), revisions)) - require.Len(t, revisions.Items, 1) - revision := revisions.Items[0].DeepCopy() - revision.Data.Raw = []byte("not-json") - - got, err := controllerRevisionMatchesTemplate(revision, &fixture.ds.Spec.Template) - require.Error(t, err) - assert.False(t, got) -} - -func TestFallbackCandidatesFailClosedAndSortReservations(t *testing.T) { - now := time.Now() - ds := &appsv1.DaemonSet{Spec: appsv1.DaemonSetSpec{MinReadySeconds: 0}} - pending := func(name, node, reservation string) corev1.Pod { - pod := pendingPodForNode(node) - pod.Name = name - pod.UID = types.UID(name + "-uid") - pod.Labels = map[string]string{appsv1.DefaultDaemonSetUniqueLabelKey: "new"} - pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: corev1.PodReasonUnschedulable, Message: "0/1 nodes are available: 1 Insufficient cpu."}} - if reservation != "" { - pod.Annotations = map[string]string{resourceFallbackOldPodAnnotation: reservation} - } - return *pod - } - old := func(name, uid, node string) corev1.Pod { - pod := readyPod(name, uid, node, "old", now.Add(-time.Minute)) - return *pod - } - - noTarget := pending("no-target", "node-a", "") - noTarget.Spec.Affinity = nil - assert.Empty(t, fallbackCandidates(ds, []corev1.Pod{noTarget, old("old-a", "old-a-uid", "node-a")}, "new", now)) - assert.Empty(t, fallbackCandidates(ds, []corev1.Pod{pending("no-old", "node-a", "")}, "new", now)) - assert.Empty(t, fallbackCandidates(ds, []corev1.Pod{ - pending("two-old", "node-a", ""), old("old-a", "old-a-uid", "node-a"), old("old-b", "old-b-uid", "node-a"), - }, "new", now)) - assert.Empty(t, fallbackCandidates(ds, []corev1.Pod{ - pending("wrong-reservation", "node-a", "other-uid"), old("old-a", "old-a-uid", "node-a"), - }, "new", now)) - - pods := []corev1.Pod{ - pending("new-c", "node-c", ""), old("old-c", "old-c-uid", "node-c"), - pending("new-b", "node-b", ""), old("old-b", "old-b-uid", "node-b"), - pending("new-a", "node-a", "old-a-uid"), old("old-a", "old-a-uid", "node-a"), - } - candidates := fallbackCandidates(ds, pods, "new", now) - require.Len(t, candidates, 3) - assert.True(t, candidates[0].reserved) - assert.Equal(t, "node-a", candidates[0].nodeName) - assert.Equal(t, "node-b", candidates[1].nodeName) - assert.Equal(t, "node-c", candidates[2].nodeName) -} - -func TestRevalidateFallbackCandidateRejectsStaleStateAndReadErrors(t *testing.T) { - candidateFor := func(fixture fallbackTestFixture) fallbackCandidate { - return fallbackCandidate{pending: fixture.pending.DeepCopy(), old: fixture.old.DeepCopy(), nodeName: "node-a", shortage: resourceShortage{cpu: true}} - } - - t.Run("DaemonSet read error", func(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - reader := interceptor.NewClient(fixture.client.(client.WithWatch), interceptor.Funcs{ - Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { - return errors.New("read failed") - }, - }) - got, err := fixture.reconciler.revalidateFallbackCandidate(context.Background(), reader, fixture.ds, candidateFor(fixture), "new-revision", false) - require.ErrorContains(t, err, "read failed") - assert.Nil(t, got) - }) - - t.Run("stale DaemonSet", func(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - expected := fixture.ds.DeepCopy() - expected.UID = "stale-uid" - got, err := fixture.reconciler.revalidateFallbackCandidate(context.Background(), fixture.client, expected, candidateFor(fixture), "new-revision", false) - require.NoError(t, err) - assert.Nil(t, got) - }) - - t.Run("revision changed", func(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - got, err := fixture.reconciler.revalidateFallbackCandidate(context.Background(), fixture.client, fixture.ds, candidateFor(fixture), "other-revision", false) - require.NoError(t, err) - assert.Nil(t, got) - }) - - for _, objectName := range []string{"new", "old"} { - t.Run("missing "+objectName+" Pod", func(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - pod := fixture.pending - if objectName == "old" { - pod = fixture.old - } - require.NoError(t, fixture.client.Delete(context.Background(), pod)) - got, err := fixture.reconciler.revalidateFallbackCandidate(context.Background(), fixture.client, fixture.ds, candidateFor(fixture), "new-revision", false) require.NoError(t, err) - assert.Nil(t, got) + assert.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{})) }) } - - t.Run("stale Pod UID", func(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - candidate := candidateFor(fixture) - candidate.pending.UID = "stale-pending" - got, err := fixture.reconciler.revalidateFallbackCandidate(context.Background(), fixture.client, fixture.ds, candidate, "new-revision", false) - require.NoError(t, err) - assert.Nil(t, got) - }) - - t.Run("reservation required", func(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - got, err := fixture.reconciler.revalidateFallbackCandidate(context.Background(), fixture.client, fixture.ds, candidateFor(fixture), "new-revision", true) - require.NoError(t, err) - assert.Nil(t, got) - }) - - t.Run("node disappeared", func(t *testing.T) { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - node := &corev1.Node{} - require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKey{Name: "node-a"}, node)) - require.NoError(t, fixture.client.Delete(context.Background(), node)) - got, err := fixture.reconciler.revalidateFallbackCandidate(context.Background(), fixture.client, fixture.ds, candidateFor(fixture), "new-revision", false) - require.NoError(t, err) - assert.Nil(t, got) - }) - - t.Run("Pod list errors", func(t *testing.T) { - for _, failCall := range []int{1, 2} { - fixture := newFallbackTestFixture(t, healthyNodeConditions()) - podLists := 0 - reader := interceptor.NewClient(fixture.client.(client.WithWatch), interceptor.Funcs{ - List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { - if _, ok := list.(*corev1.PodList); ok { - podLists++ - if podLists == failCall { - return errors.New("list failed") - } - } - return c.List(ctx, list, opts...) - }, - }) - got, err := fixture.reconciler.revalidateFallbackCandidate(context.Background(), reader, fixture.ds, candidateFor(fixture), "new-revision", false) - require.ErrorContains(t, err, "list") - assert.Nil(t, got) - } - }) } -func TestToleratesBlockingNodeTaints(t *testing.T) { - taints := []corev1.Taint{ - {Key: "dedicated", Value: "agents", Effect: corev1.TaintEffectNoSchedule}, - {Key: "draining", Effect: corev1.TaintEffectNoExecute}, - {Key: "soft", Effect: corev1.TaintEffectPreferNoSchedule}, +func TestResourceOnlyUnschedulableParsing(t *testing.T) { + pod := &corev1.Pod{Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{ + Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: corev1.PodReasonUnschedulable, + Message: "0/3 nodes are available: 1 Insufficient cpu, 2 node(s) didn't match Pod's node affinity/selector. preemption: 0/3 nodes are available", + }}}} + shortage, ok := resourceOnlyUnschedulable(pod) + assert.True(t, ok) + assert.True(t, shortage.cpu) + assert.False(t, shortage.memory) + pod.Status.Conditions[0].Message = "0/1 nodes are available: 1 node(s) had untolerated taint." + _, ok = resourceOnlyUnschedulable(pod) + assert.False(t, ok) +} + +func TestResourceFallbackRejectsExistingPodAntiAffinity(t *testing.T) { + fixture := newResourceFallbackFixture(t) + blocker := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "blocker", Namespace: fixture.pending.Namespace, Labels: map[string]string{"app": "other"}}, + Spec: corev1.PodSpec{ + NodeName: "node-a", + Affinity: &corev1.Affinity{PodAntiAffinity: &corev1.PodAntiAffinity{RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "agent"}}, + TopologyKey: corev1.LabelHostname, + }}}}, + Containers: []corev1.Container{{Name: "blocker"}}, + }, } - assert.False(t, toleratesBlockingNodeTaints(nil, taints)) - assert.False(t, toleratesBlockingNodeTaints([]corev1.Toleration{ - {Key: "dedicated", Value: "agents", Operator: corev1.TolerationOpEqual, Effect: corev1.TaintEffectNoSchedule}, - }, taints)) - assert.True(t, toleratesBlockingNodeTaints([]corev1.Toleration{ - {Key: "dedicated", Value: "agents", Operator: corev1.TolerationOpEqual, Effect: corev1.TaintEffectNoSchedule}, - {Key: "draining", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoExecute}, - }, taints)) - assert.True(t, toleratesBlockingNodeTaints([]corev1.Toleration{{Operator: corev1.TolerationOpExists}}, taints)) -} + require.NoError(t, fixture.client.Create(context.Background(), blocker)) -func TestNodeReadyForResourceFallbackFailsClosed(t *testing.T) { - ready := &corev1.Node{Status: corev1.NodeStatus{Conditions: healthyNodeConditions()}} - assert.True(t, nodeReadyForResourceFallback(ready)) - - unschedulable := ready.DeepCopy() - unschedulable.Spec.Unschedulable = true - assert.False(t, nodeReadyForResourceFallback(unschedulable)) - - deleting := ready.DeepCopy() - deleting.DeletionTimestamp = &metav1.Time{Time: time.Now()} - assert.False(t, nodeReadyForResourceFallback(deleting)) - - networkUnavailable := ready.DeepCopy() - for i := range networkUnavailable.Status.Conditions { - if networkUnavailable.Status.Conditions[i].Type == corev1.NodeNetworkUnavailable { - networkUnavailable.Status.Conditions[i].Status = corev1.ConditionTrue - } - } - assert.False(t, nodeReadyForResourceFallback(networkUnavailable)) + _, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{})) } -func TestResourceFallbackDaemonSetEligibleFailsClosed(t *testing.T) { - ds := &appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{Generation: 1}, - Spec: appsv1.DaemonSetSpec{UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ - Type: appsv1.RollingUpdateDaemonSetStrategyType, - RollingUpdate: &appsv1.RollingUpdateDaemonSet{MaxSurge: ptr.To(intstr.FromInt(1))}, - }}, - Status: appsv1.DaemonSetStatus{ObservedGeneration: 1, DesiredNumberScheduled: 1}, +func TestConsumedResourceFallbackBudgetCountsDisjointUnavailableNodes(t *testing.T) { + fixture := newResourceFallbackFixture(t) + fixture.ds.Status.DesiredNumberScheduled = 2 + fixture.ds.Status.NumberUnavailable = 1 + fixture.pending.Annotations = map[string]string{resourceFallbackOldPodAnnotation: string(fixture.old.UID)} + unavailable := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "unavailable", Namespace: "default", Labels: map[string]string{appsv1.DefaultDaemonSetUniqueLabelKey: "old"}}, + Spec: corev1.PodSpec{NodeName: "node-b"}, + Status: corev1.PodStatus{Phase: corev1.PodPending}, } - assert.True(t, resourceFallbackDaemonSetEligible(ds)) - for _, mutate := range []func(*appsv1.DaemonSet){ - func(value *appsv1.DaemonSet) { value.Status.DesiredNumberScheduled = 0 }, - func(value *appsv1.DaemonSet) { value.Status.ObservedGeneration = 0 }, - func(value *appsv1.DaemonSet) { value.Spec.UpdateStrategy.Type = appsv1.OnDeleteDaemonSetStrategyType }, - func(value *appsv1.DaemonSet) { - value.Spec.UpdateStrategy.RollingUpdate.MaxSurge = ptr.To(intstr.FromInt(0)) - }, - } { - copy := ds.DeepCopy() - mutate(copy) - assert.False(t, resourceFallbackDaemonSetEligible(copy)) - } + consumed := consumedResourceFallbackBudget(fixture.ds, []corev1.Pod{*fixture.old, *fixture.pending, unavailable}, "new", time.Now()) + assert.Equal(t, 2, consumed) } -type fallbackTestFixture struct { +type resourceFallbackFixture struct { client client.Client reconciler *Reconciler ddai *datadoghqv1alpha1.DatadogAgentInternal @@ -953,7 +132,7 @@ type fallbackTestFixture struct { pending *corev1.Pod } -func newFallbackTestFixture(t *testing.T, nodeConditions []corev1.NodeCondition) fallbackTestFixture { +func newResourceFallbackFixture(t *testing.T) resourceFallbackFixture { t.Helper() scheme := runtime.NewScheme() require.NoError(t, corev1.AddToScheme(scheme)) @@ -961,80 +140,73 @@ func newFallbackTestFixture(t *testing.T, nodeConditions []corev1.NodeCondition) require.NoError(t, datadoghqv1alpha1.AddToScheme(scheme)) ddai := &datadoghqv1alpha1.DatadogAgentInternal{ObjectMeta: metav1.ObjectMeta{Name: "agent", Namespace: "default", UID: "ddai-uid"}} - ds := testFallbackDaemonSet(t, ddai) - old := readyPod("old", "old-uid", "node-a", "old-revision", time.Now().Add(-time.Minute)) - pending := pendingPodForNode("node-a") - pending.ObjectMeta = metav1.ObjectMeta{Name: "new", Namespace: "default", UID: "new-uid", Labels: map[string]string{"app": "agent", appsv1.DefaultDaemonSetUniqueLabelKey: "new-revision"}, OwnerReferences: []metav1.OwnerReference{daemonSetOwner(ds)}} - pending.Spec.Containers = []corev1.Container{{Name: "agent", Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}}}} - pending.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: corev1.PodReasonUnschedulable, Message: "0/1 nodes are available: 1 Insufficient cpu."}} - old.Namespace = "default" - old.Labels["app"] = "agent" - old.OwnerReferences = []metav1.OwnerReference{daemonSetOwner(ds)} - old.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = resource.MustParse("1") - node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-a"}, Status: corev1.NodeStatus{Allocatable: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1500m"), corev1.ResourceMemory: resource.MustParse("1Gi"), corev1.ResourcePods: resource.MustParse("10")}, Conditions: nodeConditions}} - revision := controllerRevisionForTemplate(t, ds, "new-revision") - - c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(ddai, ds, old, pending, node, revision).WithIndex(&corev1.Pod{}, apiPodNodeNameField, func(obj client.Object) []string { + ds := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: "agent", Namespace: "default", UID: "ds-uid", Generation: 2, OwnerReferences: []metav1.OwnerReference{{ + APIVersion: datadoghqv1alpha1.GroupVersion.String(), Kind: "DatadogAgentInternal", Name: ddai.Name, UID: ddai.UID, Controller: ptr.To(true), + }}}, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "agent"}}, + Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "agent"}, Annotations: map[string]string{preparedRolloutModeAnnotation: preparedRolloutModeV1}, + }}, + UpdateStrategy: appsv1.DaemonSetUpdateStrategy{Type: appsv1.RollingUpdateDaemonSetStrategyType, RollingUpdate: &appsv1.RollingUpdateDaemonSet{ + MaxSurge: ptr.To(intstr.FromInt(1)), MaxUnavailable: ptr.To(intstr.FromInt(0)), + }}, + }, + Status: appsv1.DaemonSetStatus{ObservedGeneration: 2, DesiredNumberScheduled: 1, NumberReady: 1, NumberAvailable: 1}, + } + owner := metav1.OwnerReference{APIVersion: appsv1.SchemeGroupVersion.String(), Kind: "DaemonSet", Name: ds.Name, UID: ds.UID, Controller: ptr.To(true)} + old := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "old", Namespace: "default", UID: "old-uid", Labels: map[string]string{"app": "agent", appsv1.DefaultDaemonSetUniqueLabelKey: "old"}, OwnerReferences: []metav1.OwnerReference{owner}}, + Spec: corev1.PodSpec{NodeName: "node-a", Containers: []corev1.Container{{Name: "agent", Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}}}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue, LastTransitionTime: metav1.NewTime(time.Now().Add(-time.Minute))}}}, + } + pending := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "new", Namespace: "default", UID: "new-uid", Labels: map[string]string{"app": "agent", appsv1.DefaultDaemonSetUniqueLabelKey: "new"}, OwnerReferences: []metav1.OwnerReference{owner}}, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchFields: []corev1.NodeSelectorRequirement{{Key: metav1.ObjectNameField, Operator: corev1.NodeSelectorOpIn, Values: []string{"node-a"}}}}}}}}, + Containers: []corev1.Container{{Name: "agent", Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}}}}, + }, + Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: corev1.PodReasonUnschedulable, Message: "0/1 nodes are available: 1 Insufficient cpu."}}}, + } + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "node-a"}, + Status: corev1.NodeStatus{ + Allocatable: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1500m"), corev1.ResourceMemory: resource.MustParse("1Gi"), corev1.ResourcePods: resource.MustParse("10")}, + Conditions: []corev1.NodeCondition{ + {Type: corev1.NodeReady, Status: corev1.ConditionTrue}, + {Type: corev1.NodeMemoryPressure, Status: corev1.ConditionFalse}, + {Type: corev1.NodeDiskPressure, Status: corev1.ConditionFalse}, + {Type: corev1.NodePIDPressure, Status: corev1.ConditionFalse}, + {Type: corev1.NodeNetworkUnavailable, Status: corev1.ConditionFalse}, + }, + }, + } + revision := controllerRevisionForFallbackTest(t, ds, "new") + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&corev1.Pod{}, &appsv1.DaemonSet{}).WithObjects(ddai, ds, old, pending, node, revision).WithIndex(&corev1.Pod{}, "spec.nodeName", func(obj client.Object) []string { pod := obj.(*corev1.Pod) if pod.Spec.NodeName == "" { return nil } return []string{pod.Spec.NodeName} }).Build() - r := &Reconciler{client: c, apiReader: c} - return fallbackTestFixture{client: c, reconciler: r, ddai: ddai, ds: ds, old: old, pending: pending} + return resourceFallbackFixture{client: c, reconciler: &Reconciler{client: c, apiReader: c}, ddai: ddai, ds: ds, old: old, pending: pending} } -func healthyNodeConditions() []corev1.NodeCondition { - return []corev1.NodeCondition{ - {Type: corev1.NodeReady, Status: corev1.ConditionTrue}, - {Type: corev1.NodeMemoryPressure, Status: corev1.ConditionFalse}, - {Type: corev1.NodeDiskPressure, Status: corev1.ConditionFalse}, - {Type: corev1.NodePIDPressure, Status: corev1.ConditionFalse}, - {Type: corev1.NodeNetworkUnavailable, Status: corev1.ConditionFalse}, - } -} - -func testFallbackDaemonSet(t *testing.T, ddai *datadoghqv1alpha1.DatadogAgentInternal) *appsv1.DaemonSet { +func controllerRevisionForFallbackTest(t *testing.T, ds *appsv1.DaemonSet, hash string) *appsv1.ControllerRevision { t.Helper() - return &appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{Name: "agent", Namespace: "default", UID: "ds-uid", Generation: 2, OwnerReferences: []metav1.OwnerReference{{APIVersion: datadoghqv1alpha1.GroupVersion.String(), Kind: "DatadogAgentInternal", Name: ddai.Name, UID: ddai.UID, Controller: ptr.To(true)}}}, - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "agent"}}, - Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "agent"}}, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "agent"}}}}, - UpdateStrategy: appsv1.DaemonSetUpdateStrategy{Type: appsv1.RollingUpdateDaemonSetStrategyType, RollingUpdate: &appsv1.RollingUpdateDaemonSet{MaxSurge: ptr.To(intstr.FromInt(1)), MaxUnavailable: ptr.To(intstr.FromInt(0))}}, - }, - Status: appsv1.DaemonSetStatus{ObservedGeneration: 2, DesiredNumberScheduled: 1}, - } -} - -func controllerRevisionForTemplate(t *testing.T, ds *appsv1.DaemonSet, hash string) *appsv1.ControllerRevision { - t.Helper() - templateJSON, err := json.Marshal(ds.Spec.Template) + template, err := json.Marshal(ds.Spec.Template) require.NoError(t, err) - var templatePatch map[string]any - require.NoError(t, json.Unmarshal(templateJSON, &templatePatch)) - templatePatch["$patch"] = "replace" - data, err := json.Marshal(map[string]any{"spec": map[string]any{"template": templatePatch}}) + var patch map[string]any + require.NoError(t, json.Unmarshal(template, &patch)) + patch["$patch"] = "replace" + raw, err := json.Marshal(map[string]any{"spec": map[string]any{"template": patch}}) require.NoError(t, err) - return &appsv1.ControllerRevision{ObjectMeta: metav1.ObjectMeta{Name: "agent-" + hash, Namespace: ds.Namespace, UID: types.UID("revision-uid"), Labels: map[string]string{appsv1.DefaultDaemonSetUniqueLabelKey: hash}, OwnerReferences: []metav1.OwnerReference{daemonSetOwner(ds)}}, Revision: 2, Data: runtime.RawExtension{Raw: data}} -} - -func daemonSetOwner(ds *appsv1.DaemonSet) metav1.OwnerReference { - return metav1.OwnerReference{APIVersion: appsv1.SchemeGroupVersion.String(), Kind: "DaemonSet", Name: ds.Name, UID: ds.UID, Controller: ptr.To(true)} -} - -func pendingPodForNode(nodeName string) *corev1.Pod { - return &corev1.Pod{Spec: corev1.PodSpec{Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchFields: []corev1.NodeSelectorRequirement{{Key: metav1.ObjectNameField, Operator: corev1.NodeSelectorOpIn, Values: []string{nodeName}}}}}}}}}} -} - -func scheduledResourcePod(name, uid, nodeName, cpu, memory string) *corev1.Pod { - return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(uid)}, Spec: corev1.PodSpec{NodeName: nodeName, Containers: []corev1.Container{{Name: "agent", Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse(cpu), corev1.ResourceMemory: resource.MustParse(memory)}}}}}} -} - -func readyPod(name, uid, nodeName, revision string, readyAt time.Time) *corev1.Pod { - pod := scheduledResourcePod(name, uid, nodeName, "100m", "128Mi") - pod.Labels = map[string]string{appsv1.DefaultDaemonSetUniqueLabelKey: revision} - pod.Status = corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue, LastTransitionTime: metav1.NewTime(readyAt)}}} - return pod + return &appsv1.ControllerRevision{ + ObjectMeta: metav1.ObjectMeta{Name: "agent-new", Namespace: ds.Namespace, UID: types.UID("revision-uid"), Labels: map[string]string{appsv1.DefaultDaemonSetUniqueLabelKey: hash}, OwnerReferences: []metav1.OwnerReference{{ + APIVersion: appsv1.SchemeGroupVersion.String(), Kind: "DaemonSet", Name: ds.Name, UID: ds.UID, Controller: ptr.To(true), + }}}, + Revision: 2, + Data: runtime.RawExtension{Raw: raw}, + } } diff --git a/internal/controller/datadogagentinternal_controller.go b/internal/controller/datadogagentinternal_controller.go index 3f7e2bf855..fb373f7832 100644 --- a/internal/controller/datadogagentinternal_controller.go +++ b/internal/controller/datadogagentinternal_controller.go @@ -37,6 +37,8 @@ import ( "github.com/DataDog/datadog-operator/pkg/kubernetes" ) +const preparedRolloutModeAnnotationKey = "experimental.agent.datadoghq.com/node-agent-rollout-mode" + // DatadogAgentInternalReconciler reconciles a DatadogAgentInternal object. type DatadogAgentInternalReconciler struct { client.Client @@ -86,7 +88,7 @@ func (r *DatadogAgentInternalReconciler) SetupWithManager(mgr ctrl.Manager, metr builder.Watches( &corev1.Pod{}, handler.EnqueueRequestsFromMapFunc(enqueueDatadogAgentInternalForPod(mgr.GetAPIReader())), - ctrlbuilder.WithPredicates(resourceFallbackPodPredicate()), + ctrlbuilder.WithPredicates(preparedRolloutPodPredicate()), ) if r.Options.ExtendedDaemonsetOptions.Enabled { @@ -148,6 +150,9 @@ func enqueueDatadogAgentInternalForPod(reader client.Reader) handler.MapFunc { if err := reader.Get(ctx, client.ObjectKey{Namespace: pod.Namespace, Name: podOwner.Name}, ds); err != nil || ds.UID != podOwner.UID { return nil } + if ds.Spec.Template.Annotations[preparedRolloutModeAnnotationKey] != "prepared-surge-v1" { + return nil + } ddaiOwner := metav1.GetControllerOf(ds) if ddaiOwner == nil || ddaiOwner.APIVersion != datadoghqv1alpha1.GroupVersion.String() || ddaiOwner.Kind != "DatadogAgentInternal" { return nil @@ -156,11 +161,11 @@ func enqueueDatadogAgentInternalForPod(reader client.Reader) handler.MapFunc { } } -func resourceFallbackPodPredicate() predicate.Predicate { +func preparedRolloutPodPredicate() predicate.Predicate { return predicate.Funcs{ CreateFunc: func(e event.CreateEvent) bool { - pod, ok := e.Object.(*corev1.Pod) - return ok && resourceFallbackSchedulingCondition(pod) != nil + _, ok := e.Object.(*corev1.Pod) + return ok }, UpdateFunc: func(e event.UpdateEvent) bool { oldPod, oldOK := e.ObjectOld.(*corev1.Pod) @@ -168,8 +173,8 @@ func resourceFallbackPodPredicate() predicate.Predicate { if !oldOK || !newOK { return false } - return resourceFallbackConditionChanged(oldPod, newPod, corev1.PodScheduled) || - resourceFallbackConditionChanged(oldPod, newPod, corev1.PodReady) || + return podConditionChanged(oldPod, newPod, corev1.PodScheduled) || + podConditionChanged(oldPod, newPod, corev1.PodReady) || containerRolloutStatusChanged(oldPod.Status.InitContainerStatuses, newPod.Status.InitContainerStatuses) || containerRolloutStatusChanged(oldPod.Status.ContainerStatuses, newPod.Status.ContainerStatuses) }, @@ -197,7 +202,7 @@ func containerRolloutStatusChanged(oldStatuses, newStatuses []corev1.ContainerSt return false } -func resourceFallbackConditionChanged(oldPod, newPod *corev1.Pod, conditionType corev1.PodConditionType) bool { +func podConditionChanged(oldPod, newPod *corev1.Pod, conditionType corev1.PodConditionType) bool { oldCondition := podCondition(oldPod, conditionType) newCondition := podCondition(newPod, conditionType) if oldCondition == nil || newCondition == nil { @@ -215,10 +220,6 @@ func podCondition(pod *corev1.Pod, conditionType corev1.PodConditionType) *corev return nil } -func resourceFallbackSchedulingCondition(pod *corev1.Pod) *corev1.PodCondition { - return podCondition(pod, corev1.PodScheduled) -} - func enqueueIfOwnedByDatadogAgentInternal(ctx context.Context, obj client.Object) []reconcile.Request { labels := obj.GetLabels() diff --git a/internal/controller/datadogagentinternal_controller_test.go b/internal/controller/datadogagentinternal_controller_test.go index 27f4aaf494..81c5226935 100644 --- a/internal/controller/datadogagentinternal_controller_test.go +++ b/internal/controller/datadogagentinternal_controller_test.go @@ -91,25 +91,20 @@ func TestDatadogAgentInternalSetupWithManager(t *testing.T) { } } -func TestResourceFallbackPodPredicate(t *testing.T) { - predicate := resourceFallbackPodPredicate() +func TestPreparedRolloutPodPredicate(t *testing.T) { + predicate := preparedRolloutPodPredicate() assert.False(t, predicate.Create(event.CreateEvent{Object: &corev1.ConfigMap{}})) - assert.False(t, predicate.Create(event.CreateEvent{Object: &corev1.Pod{}})) - oldPod := &corev1.Pod{Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: corev1.PodReasonUnschedulable, Message: "old"}}}} + oldPod := &corev1.Pod{} assert.True(t, predicate.Create(event.CreateEvent{Object: oldPod})) - newPod := oldPod.DeepCopy() - newPod.Status.Conditions[0].Message = "0/1 nodes are available: 1 Insufficient cpu." - assert.True(t, predicate.Update(event.UpdateEvent{ObjectOld: oldPod, ObjectNew: newPod}), "PodScheduled message-only updates must enqueue") - - readyPod := newPod.DeepCopy() + readyPod := oldPod.DeepCopy() readyPod.Status.Conditions = append(readyPod.Status.Conditions, corev1.PodCondition{Type: corev1.PodReady, Status: corev1.ConditionTrue}) - assert.True(t, predicate.Update(event.UpdateEvent{ObjectOld: newPod, ObjectNew: readyPod}), "PodReady transitions must release fallback reservations promptly") + assert.True(t, predicate.Update(event.UpdateEvent{ObjectOld: oldPod, ObjectNew: readyPod}), "PodReady transitions must enqueue capacity fallback") waiting := readyPod.DeepCopy() waiting.Status.ContainerStatuses = []corev1.ContainerStatus{{Name: "agent", Started: ptr.To(false), State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}} prepared := waiting.DeepCopy() prepared.Status.ContainerStatuses[0].Started = ptr.To(true) - assert.True(t, predicate.Update(event.UpdateEvent{ObjectOld: waiting, ObjectNew: prepared}), "startup-probe Prepared transitions must enqueue the handoff controller") + assert.True(t, predicate.Update(event.UpdateEvent{ObjectOld: waiting, ObjectNew: prepared}), "startup-probe transitions must enqueue capacity fallback") assert.False(t, predicate.Update(event.UpdateEvent{ObjectOld: &corev1.ConfigMap{}, ObjectNew: &corev1.ConfigMap{}})) assert.True(t, predicate.Delete(event.DeleteEvent{Object: oldPod})) assert.False(t, predicate.Generic(event.GenericEvent{Object: oldPod})) @@ -140,20 +135,20 @@ func TestContainerRolloutStatusChanged(t *testing.T) { } } -func TestResourceFallbackConditionChanged(t *testing.T) { +func TestPodConditionChanged(t *testing.T) { empty := &corev1.Pod{} - scheduled := &corev1.Pod{Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: "Unschedulable", Message: "cpu"}}}} - assert.False(t, resourceFallbackConditionChanged(empty, empty, corev1.PodScheduled)) - assert.True(t, resourceFallbackConditionChanged(empty, scheduled, corev1.PodScheduled)) - assert.False(t, resourceFallbackConditionChanged(scheduled, scheduled.DeepCopy(), corev1.PodScheduled)) + ready := &corev1.Pod{Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionFalse, Reason: "Starting"}}}} + assert.False(t, podConditionChanged(empty, empty, corev1.PodReady)) + assert.True(t, podConditionChanged(empty, ready, corev1.PodReady)) + assert.False(t, podConditionChanged(ready, ready.DeepCopy(), corev1.PodReady)) for _, mutate := range []func(*corev1.PodCondition){ func(condition *corev1.PodCondition) { condition.Status = corev1.ConditionTrue }, - func(condition *corev1.PodCondition) { condition.Reason = "Scheduled" }, - func(condition *corev1.PodCondition) { condition.Message = "memory" }, + func(condition *corev1.PodCondition) { condition.Reason = "Ready" }, + func(condition *corev1.PodCondition) { condition.Message = "healthy" }, } { - changed := scheduled.DeepCopy() + changed := ready.DeepCopy() mutate(&changed.Status.Conditions[0]) - assert.True(t, resourceFallbackConditionChanged(scheduled, changed, corev1.PodScheduled)) + assert.True(t, podConditionChanged(ready, changed, corev1.PodReady)) } } @@ -166,17 +161,9 @@ func TestDatadogAgentInternalEventPredicate(t *testing.T) { assert.True(t, p.Update(event.UpdateEvent{ObjectOld: old, ObjectNew: generation})) prepared := old.DeepCopy() - prepared.Annotations["experimental.agent.datadoghq.com/host-network-surge-prepared"] = "true" + prepared.Annotations[preparedRolloutModeAnnotationKey] = "prepared-surge-v1" assert.True(t, p.Update(event.UpdateEvent{ObjectOld: old, ObjectNew: prepared})) - fallback := prepared.DeepCopy() - fallback.Annotations["experimental.agent.datadoghq.com/resource-fallback"] = "true" - assert.True(t, p.Update(event.UpdateEvent{ObjectOld: prepared, ObjectNew: fallback})) - - removed := fallback.DeepCopy() - delete(removed.Annotations, "experimental.agent.datadoghq.com/resource-fallback") - assert.True(t, p.Update(event.UpdateEvent{ObjectOld: fallback, ObjectNew: removed})) - unrelated := old.DeepCopy() unrelated.Annotations["example.com/ignored"] = "new" assert.False(t, p.Update(event.UpdateEvent{ObjectOld: old, ObjectNew: unrelated})) @@ -196,7 +183,7 @@ func TestEnqueueDatadogAgentInternalForPodFollowsDaemonSetOwner(t *testing.T) { UID: types.UID("ddai-uid"), Controller: ptr.To(true), }}, - }} + }, Spec: appsv1.DaemonSetSpec{Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{preparedRolloutModeAnnotationKey: "prepared-surge-v1"}}}}} reader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(ds).Build() pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ Name: "profile-agent-new", @@ -216,6 +203,16 @@ func TestEnqueueDatadogAgentInternalForPodFollowsDaemonSetOwner(t *testing.T) { assert.Equal(t, "default", requests[0].Namespace) assert.Equal(t, "profile-ddai", requests[0].Name) + ordinaryDS := ds.DeepCopy() + ordinaryDS.Name = "ordinary-agent" + ordinaryDS.UID = "ordinary-ds-uid" + ordinaryDS.Spec.Template.Annotations = nil + ordinaryReader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(ordinaryDS).Build() + ordinaryPod := pod.DeepCopy() + ordinaryPod.OwnerReferences[0].Name = ordinaryDS.Name + ordinaryPod.OwnerReferences[0].UID = ordinaryDS.UID + assert.Empty(t, enqueueDatadogAgentInternalForPod(ordinaryReader)(context.Background(), ordinaryPod), "ordinary Agent Pods must not trigger prepared-rollout reconciles") + pod.OwnerReferences[0].UID = "wrong-uid" assert.Empty(t, enqueueDatadogAgentInternalForPod(reader)(context.Background(), pod), "stale Pod owner UIDs must not enqueue") diff --git a/internal/controller/testutils/renderer/render_e2e_test.go b/internal/controller/testutils/renderer/render_e2e_test.go index 14cd89c98b..0dd1193eca 100644 --- a/internal/controller/testutils/renderer/render_e2e_test.go +++ b/internal/controller/testutils/renderer/render_e2e_test.go @@ -165,7 +165,7 @@ func TestRender_AppArmorProfileVersionGate(t *testing.T) { } } -func TestRender_PreparedRolloutArmsBeforeSurge(t *testing.T) { +func TestRender_PreparedRolloutUsesNativeSurge(t *testing.T) { renderAgentDaemonSet := func(t *testing.T, prepared bool) *appsv1.DaemonSet { t.Helper() dda, err := LoadDDA("testdata/minimal-dda.yaml") @@ -181,7 +181,7 @@ func TestRender_PreparedRolloutArmsBeforeSurge(t *testing.T) { }, } if prepared { - dda.Annotations = map[string]string{"experimental.agent.datadoghq.com/host-network-surge-prepared": "true"} + dda.Annotations = map[string]string{"experimental.agent.datadoghq.com/node-agent-rollout-mode": "prepared-surge-v1"} } dda.Spec.Override = map[datadoghqv2alpha1.ComponentName]*datadoghqv2alpha1.DatadogAgentComponentOverride{ datadoghqv2alpha1.NodeAgentComponentName: override, @@ -210,15 +210,15 @@ func TestRender_PreparedRolloutArmsBeforeSurge(t *testing.T) { } require.Positive(t, baselinePortCount, "the host-network baseline must exercise Kubernetes's implicit hostPort defaulting") - armed := renderAgentDaemonSet(t, true) - require.True(t, armed.Spec.Template.Spec.HostNetwork) - require.NotNil(t, armed.Spec.UpdateStrategy.RollingUpdate) - assert.Equal(t, intstr.FromInt(0), *armed.Spec.UpdateStrategy.RollingUpdate.MaxSurge) - assert.Equal(t, intstr.FromInt(1), *armed.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable) - assert.Equal(t, "arm", armed.Spec.Template.Annotations["experimental.agent.datadoghq.com/prepared-rollout-phase"]) - armedPortCount := 0 - for _, container := range armed.Spec.Template.Spec.Containers { - armedPortCount += len(container.Ports) + prepared := renderAgentDaemonSet(t, true) + require.True(t, prepared.Spec.Template.Spec.HostNetwork) + require.NotNil(t, prepared.Spec.UpdateStrategy.RollingUpdate) + assert.Equal(t, intstr.FromInt(1), *prepared.Spec.UpdateStrategy.RollingUpdate.MaxSurge) + assert.Equal(t, intstr.FromInt(0), *prepared.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable) + assert.Equal(t, "prepared-surge-v1", prepared.Spec.Template.Annotations["experimental.agent.datadoghq.com/node-agent-rollout-mode"]) + preparedPortCount := 0 + for _, container := range prepared.Spec.Template.Spec.Containers { + preparedPortCount += len(container.Ports) require.NotNil(t, container.StartupProbe) require.NotNil(t, container.StartupProbe.Exec) require.NotNil(t, container.ReadinessProbe) @@ -227,14 +227,14 @@ func TestRender_PreparedRolloutArmsBeforeSurge(t *testing.T) { assert.Equal(t, "trace-agent", container.Command[0], "prepared mode must bypass trace-loader") } } - assert.Equal(t, baselinePortCount, armedPortCount, "arming is a conventional rollout and keeps port declarations") + assert.Zero(t, preparedPortCount, "host-network replacements must not claim ports in the scheduler") } func TestRender_PreparedHostNetworkSurgeWithProfiles(t *testing.T) { dda, err := LoadDDA("testdata/minimal-dda.yaml") require.NoError(t, err) dda.Spec.Features = preparedRolloutTestFeatures() - dda.Annotations = map[string]string{"experimental.agent.datadoghq.com/host-network-surge-prepared": "true"} + dda.Annotations = map[string]string{"experimental.agent.datadoghq.com/node-agent-rollout-mode": "prepared-surge-v1"} surgeOverride := func() *datadoghqv2alpha1.DatadogAgentComponentOverride { return &datadoghqv2alpha1.DatadogAgentComponentOverride{ @@ -265,7 +265,7 @@ func TestRender_PreparedHostNetworkSurgeWithProfiles(t *testing.T) { } daemonSets++ require.True(t, ds.Spec.Template.Spec.HostNetwork) - assert.Equal(t, "arm", ds.Spec.Template.Annotations["experimental.agent.datadoghq.com/prepared-rollout-phase"]) + assert.Equal(t, "prepared-surge-v1", ds.Spec.Template.Annotations["experimental.agent.datadoghq.com/node-agent-rollout-mode"]) require.NotNil(t, ds.Spec.Template.Spec.Affinity) require.NotNil(t, ds.Spec.Template.Spec.Affinity.PodAntiAffinity) assert.Len(t, ds.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, 2, From 688bc91038a95784270ba93c6aa61cff21268a7b Mon Sep 17 00:00:00 2001 From: "ali.benabdallah" Date: Tue, 28 Jul 2026 13:35:51 +0200 Subject: [PATCH 12/16] Delete old Agent pods after surged containers start --- .../controller_reconcile_agent.go | 2 +- .../controller_reconcile_agent_test.go | 4 +- .../datadogagentinternal/prepared_rollout.go | 199 ++--- .../prepared_rollout_support.go | 3 +- .../prepared_rollout_test.go | 211 ++---- .../datadogagentinternal/resource_fallback.go | 706 ++++++------------ .../resource_fallback_test.go | 303 +++++--- .../datadogagentinternal_controller.go | 100 +-- .../datadogagentinternal_controller_test.go | 230 +----- .../testutils/renderer/render_e2e_test.go | 14 +- 10 files changed, 594 insertions(+), 1178 deletions(-) diff --git a/internal/controller/datadogagentinternal/controller_reconcile_agent.go b/internal/controller/datadogagentinternal/controller_reconcile_agent.go index e9c148bfae..786e782d4e 100644 --- a/internal/controller/datadogagentinternal/controller_reconcile_agent.go +++ b/internal/controller/datadogagentinternal/controller_reconcile_agent.go @@ -325,7 +325,7 @@ func (r *Reconciler) reconcileV2Agent(ctx context.Context, requiredComponents fe } return result, nil } - fallbackResult, fallbackErr := r.reconcileResourceFallback(ctx, ddai, daemonset, rolloutBudget) + fallbackResult, fallbackErr := r.reconcilePreparedRollout(ctx, ddai, daemonset, rolloutBudget) if fallbackErr != nil { return reconcile.Result{}, fallbackErr } diff --git a/internal/controller/datadogagentinternal/controller_reconcile_agent_test.go b/internal/controller/datadogagentinternal/controller_reconcile_agent_test.go index fa95f12eb0..b3946e2f2d 100644 --- a/internal/controller/datadogagentinternal/controller_reconcile_agent_test.go +++ b/internal/controller/datadogagentinternal/controller_reconcile_agent_test.go @@ -32,7 +32,7 @@ const defaultProvider = kubernetes.DefaultProvider const gkeCosProvider = kubernetes.GKECloudProvider + "-" + kubernetes.GKECosType func TestReconcileV2AgentCreatesPreparedSurgeDaemonSet(t *testing.T) { - r, ddai := newPreparedRolloutReconciler(t, false) + r, ddai := newPreparedRolloutReconciler(t, true) status := &datadoghqv1alpha1.DatadogAgentInternalStatus{} result, err := r.reconcileV2Agent( @@ -46,7 +46,7 @@ func TestReconcileV2AgentCreatesPreparedSurgeDaemonSet(t *testing.T) { ) require.NoError(t, err) - assert.Zero(t, result.RequeueAfter) + assert.Equal(t, resourceFallbackPollInterval, result.RequeueAfter) daemonSets := &appsv1.DaemonSetList{} require.NoError(t, r.client.List(context.Background(), daemonSets)) require.Len(t, daemonSets.Items, 1) diff --git a/internal/controller/datadogagentinternal/prepared_rollout.go b/internal/controller/datadogagentinternal/prepared_rollout.go index adea3a9ef9..a98f1109d1 100644 --- a/internal/controller/datadogagentinternal/prepared_rollout.go +++ b/internal/controller/datadogagentinternal/prepared_rollout.go @@ -6,7 +6,7 @@ package datadogagentinternal import ( "fmt" - "path" + "slices" "strings" appsv1 "k8s.io/api/apps/v1" @@ -22,27 +22,35 @@ const ( preparedRolloutModeAnnotation = "experimental.agent.datadoghq.com/node-agent-rollout-mode" preparedRolloutModeV1 = "prepared-surge-v1" - preparedRolloutStateVolume = "agent-rollout-state" - preparedRolloutStateDir = "/var/run/datadog-agent-rollout-state" - - rolloutEnabledEnv = "DD_EXPERIMENTAL_NODE_AGENT_ROLLOUT_ENABLED" - rolloutStatePathEnv = "DD_EXPERIMENTAL_NODE_AGENT_ROLLOUT_STATE_PATH" - rolloutPodUIDEnv = "DD_EXPERIMENTAL_NODE_AGENT_ROLLOUT_POD_UID" + rolloutEnabledEnv = "DD_EXPERIMENTAL_NODE_AGENT_ROLLOUT_ENABLED" + rolloutPodUIDEnv = "DD_EXPERIMENTAL_NODE_AGENT_ROLLOUT_POD_UID" + kubeletHostEnv = "DD_KUBERNETES_KUBELET_HOST" ) -var preparedRolloutContainerNames = []string{ - string(apicommon.CoreAgentContainerName), - string(apicommon.TraceAgentContainerName), +var preparedRolloutContainerNames = map[string]struct{}{ + string(apicommon.CoreAgentContainerName): {}, + string(apicommon.TraceAgentContainerName): {}, + string(apicommon.ProcessAgentContainerName): {}, + string(apicommon.SystemProbeContainerName): {}, + string(apicommon.HostProfiler): {}, + string(apicommon.OtelAgent): {}, + string(apicommon.PrivateActionRunnerContainerName): {}, +} + +var preparedRolloutInitContainerNames = map[string]struct{}{ + string(apicommon.InitVolumeContainerName): {}, + string(apicommon.InitConfigContainerName): {}, + "seccomp-setup": {}, + "host-profiler-seccomp-setup": {}, } func preparedRolloutEnabled(ddai *datadoghqv1alpha1.DatadogAgentInternal) bool { return ddai != nil && ddai.Annotations[preparedRolloutModeAnnotation] == preparedRolloutModeV1 } -// configurePreparedRollout enables native DaemonSet surge. Profile-managed -// DaemonSets first need one conventional affinity-only rollout because an old -// Pod's broad required anti-affinity also rejects an incoming replacement. -// The returned boolean is true while that prerequisite rollout is in progress. +// configurePreparedRollout enables native DaemonSet surge. Existing profile +// DaemonSets need one ordinary rollout to narrow their anti-affinity before two +// revisions can share a node. func configurePreparedRollout(ddai *datadoghqv1alpha1.DatadogAgentInternal, ds, current *appsv1.DaemonSet, budget intstr.IntOrString) (bool, error) { if !preparedRolloutEnabled(ddai) { return false, nil @@ -63,10 +71,8 @@ func configurePreparedRollout(ddai *datadoghqv1alpha1.DatadogAgentInternal, ds, if current != nil && profileAffinityMigrationPending(current) { migrationTemplate := current.Spec.Template.DeepCopy() - if apiequality.Semantic.DeepEqual(migrationTemplate.Spec.Affinity.PodAntiAffinity, broadAgentPodAntiAffinity()) { - if !prepareProfileAntiAffinityForSurge(migrationTemplate) { - return false, fmt.Errorf("prepared Agent rollout cannot migrate profile anti-affinity") - } + if apiequality.Semantic.DeepEqual(migrationTemplate.Spec.Affinity.PodAntiAffinity, broadAgentPodAntiAffinity()) && !prepareProfileAntiAffinityForSurge(migrationTemplate) { + return false, fmt.Errorf("prepared Agent rollout cannot migrate profile anti-affinity") } ds.Spec.Template = *migrationTemplate configureConventionalMigration(ds, budget) @@ -99,10 +105,7 @@ func daemonSetFullyRolledOut(ds *appsv1.DaemonSet) bool { func prepareAgentTemplate(ds *appsv1.DaemonSet) error { spec := &ds.Spec.Template.Spec - if spec.OS != nil && spec.OS.Name != corev1.Linux { - return fmt.Errorf("prepared Agent rollout is Linux-only") - } - if spec.NodeSelector[corev1.LabelOSStable] == "windows" || spec.NodeSelector["beta.kubernetes.io/os"] == "windows" { + if spec.OS != nil && spec.OS.Name != corev1.Linux || spec.NodeSelector[corev1.LabelOSStable] == "windows" || spec.NodeSelector["beta.kubernetes.io/os"] == "windows" { return fmt.Errorf("prepared Agent rollout is Linux-only") } if err := validatePreparedContainers(spec); err != nil { @@ -114,29 +117,20 @@ func prepareAgentTemplate(ds *appsv1.DaemonSet) error { if !spec.HostNetwork && podUsesHostPorts(spec) { return fmt.Errorf("prepared Agent rollout cannot overlap Pod-networked containers that declare hostPort") } - if err := addPreparedRolloutStateVolume(spec); err != nil { - return err - } for i := range spec.Containers { container := &spec.Containers[i] if container.Name == string(apicommon.TraceAgentContainerName) { - traceIndex := -1 - for commandIndex, command := range container.Command { - if command == "trace-agent" { - traceIndex = commandIndex - break - } - } + traceIndex := slices.Index(container.Command, "trace-agent") if traceIndex < 0 { return fmt.Errorf("prepared Agent rollout cannot bypass an unknown trace-agent loader command") } container.Command = append([]string(nil), container.Command[traceIndex:]...) } configurePreparedContainer(container) - // With host networking, Kubernetes' scheduler treats declared container - // ports as node-local claims. The process can still bind the same host - // address after the older Pod exits without these declarations. if spec.HostNetwork { + // On hostNetwork, declared container ports are scheduler host-port + // claims. The process still binds the same node ports after its older + // peer stops. Pod-networked containers keep their port metadata. container.Ports = nil } } @@ -171,86 +165,55 @@ func podUsesHostPorts(spec *corev1.PodSpec) bool { } func validatePreparedContainers(spec *corev1.PodSpec) error { - if len(spec.Containers) != len(preparedRolloutContainerNames) { - return fmt.Errorf("prepared Agent rollout initially supports exactly agent and trace-agent containers") - } - seen := map[string]bool{} for i := range spec.Containers { container := &spec.Containers[i] - if container.Name != string(apicommon.CoreAgentContainerName) && container.Name != string(apicommon.TraceAgentContainerName) { + if _, ok := preparedRolloutContainerNames[container.Name]; !ok { return fmt.Errorf("prepared Agent rollout does not support container %q", container.Name) } - if seen[container.Name] { - return fmt.Errorf("prepared Agent rollout found duplicate container %q", container.Name) - } - seen[container.Name] = true if container.Lifecycle != nil { return fmt.Errorf("prepared Agent rollout does not support lifecycle hooks on container %q", container.Name) } - if len(container.Args) != 0 { - return fmt.Errorf("prepared Agent rollout does not support command arguments on container %q", container.Name) - } - if container.Name == string(apicommon.CoreAgentContainerName) && (len(container.Command) != 2 || container.Command[0] != "agent" || container.Command[1] != "run") { - return fmt.Errorf("prepared Agent rollout requires the standard agent run command") + if !preparedContainerCommandSupported(container) { + return fmt.Errorf("prepared Agent rollout does not support command %q on container %q", container.Command, container.Name) } - for _, mount := range container.VolumeMounts { - if mount.Name == preparedRolloutStateVolume || mountContainsPath(mount.MountPath, preparedRolloutStateDir) { - return fmt.Errorf("prepared Agent rollout volume mount on container %q conflicts with a reserved name or path", container.Name) - } - } - } - if !seen[string(apicommon.CoreAgentContainerName)] || !seen[string(apicommon.TraceAgentContainerName)] { - return fmt.Errorf("prepared Agent rollout requires agent and trace-agent containers") } - - if len(spec.InitContainers) != 2 { - return fmt.Errorf("prepared Agent rollout initially supports only init-volume and init-config init containers") - } - seenInit := map[string]bool{} for i := range spec.InitContainers { container := &spec.InitContainers[i] - if container.Name != string(apicommon.InitVolumeContainerName) && container.Name != string(apicommon.InitConfigContainerName) { + if _, ok := preparedRolloutInitContainerNames[container.Name]; !ok { return fmt.Errorf("prepared Agent rollout does not support init container %q", container.Name) } - if container.Lifecycle != nil || len(container.Ports) != 0 { - return fmt.Errorf("prepared Agent rollout does not support ports or lifecycle hooks on init container %q", container.Name) + if container.Lifecycle != nil { + return fmt.Errorf("prepared Agent rollout does not support lifecycle hooks on init container %q", container.Name) } - seenInit[container.Name] = true - } - if !seenInit[string(apicommon.InitVolumeContainerName)] || !seenInit[string(apicommon.InitConfigContainerName)] { - return fmt.Errorf("prepared Agent rollout requires init-volume and init-config") } return nil } -func mountContainsPath(mountPath, target string) bool { - mountPath = path.Clean(mountPath) - target = path.Clean(target) - return mountPath == "/" || target == mountPath || strings.HasPrefix(target, mountPath+"/") -} - -func addPreparedRolloutStateVolume(spec *corev1.PodSpec) error { - for i := range spec.Volumes { - if spec.Volumes[i].Name == preparedRolloutStateVolume { - return fmt.Errorf("prepared Agent rollout volume name %q is reserved", preparedRolloutStateVolume) - } +func preparedContainerCommandSupported(container *corev1.Container) bool { + if len(container.Command) == 0 { + return false } - spec.Volumes = append(spec.Volumes, corev1.Volume{ - Name: preparedRolloutStateVolume, - VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, - }) - return nil + expected := map[string]string{ + string(apicommon.CoreAgentContainerName): "agent", + string(apicommon.TraceAgentContainerName): "trace-agent", + string(apicommon.ProcessAgentContainerName): "process-agent", + string(apicommon.SystemProbeContainerName): "system-probe", + string(apicommon.HostProfiler): "host-profiler", + string(apicommon.OtelAgent): "otel-agent", + string(apicommon.PrivateActionRunnerContainerName): "/opt/datadog-agent/embedded/bin/privateactionrunner", + } + want := expected[container.Name] + if container.Name == string(apicommon.TraceAgentContainerName) { + return slices.Contains(container.Command, want) + } + if container.Command[0] != want { + return false + } + return true } func configurePreparedContainer(container *corev1.Container) { - statePath := preparedRolloutStateDir + "/" + container.Name + ".state" - originalLiveness := container.LivenessProbe.DeepCopy() - originalReadiness := container.ReadinessProbe.DeepCopy() - if originalReadiness == nil { - originalReadiness = originalLiveness - } setContainerEnv(container, corev1.EnvVar{Name: rolloutEnabledEnv, Value: "true"}) - setContainerEnv(container, corev1.EnvVar{Name: rolloutStatePathEnv, Value: statePath}) setContainerEnv(container, corev1.EnvVar{ Name: rolloutPodUIDEnv, ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{ @@ -258,10 +221,13 @@ func configurePreparedContainer(container *corev1.Container) { FieldPath: "metadata.uid", }}, }) - container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{Name: preparedRolloutStateVolume, MountPath: preparedRolloutStateDir}) - container.StartupProbe = rolloutStateProbe(statePath, "prepared|activating|active", 1, 300) - container.LivenessProbe = rolloutHealthProbe(container.Name, statePath, originalLiveness) - container.ReadinessProbe = rolloutHealthProbe(container.Name, statePath, originalReadiness) + setContainerEnv(container, corev1.EnvVar{ + Name: kubeletHostEnv, + ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "status.hostIP", + }}, + }) } func setContainerEnv(container *corev1.Container, env corev1.EnvVar) { @@ -274,47 +240,6 @@ func setContainerEnv(container *corev1.Container, env corev1.EnvVar) { container.Env = append(container.Env, env) } -func rolloutStateProbe(path, accepted string, period, failures int32) *corev1.Probe { - command := rolloutStateReadCommand(path) + fmt.Sprintf(`case "$state" in %s) exit 0;; *) exit 1;; esac`, accepted) - return &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"sh", "-c", command}}}, - PeriodSeconds: period, - TimeoutSeconds: 1, - FailureThreshold: failures, - } -} - -// rolloutHealthProbe deliberately treats Prepared as ready. That is the -// contract consumed by native DaemonSet maxSurge: images, init containers and -// the Agent graph are ready, while data-producing Fx hooks remain stopped. -func rolloutHealthProbe(containerName, statePath string, base *corev1.Probe) *corev1.Probe { - activeHealth := "exit 1" - switch containerName { - case string(apicommon.CoreAgentContainerName): - activeHealth = "exec /opt/datadog-agent/bin/agent/agent health" - case string(apicommon.TraceAgentContainerName): - activeHealth = "exec 3<>/dev/tcp/127.0.0.1/8126; exec 3>&-; exec 3<&-" - } - // Prepared may wait indefinitely for the old Pod. Activating must use the - // normal liveness failure budget so a hung Fx start is eventually restarted. - acceptedWaiting := "prepared) exit 0;; " - command := rolloutStateReadCommand(statePath) + fmt.Sprintf(`case "$state" in %sactive) %s;; *) exit 1;; esac`, acceptedWaiting, activeHealth) - - probe := &corev1.Probe{PeriodSeconds: 10, TimeoutSeconds: 1, FailureThreshold: 3} - if base != nil { - probe = base.DeepCopy() - } - probe.ProbeHandler = corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"bash", "-c", command}}} - return probe -} - -// rolloutStateReadCommand rejects state left by a previous container -// generation. EmptyDir volumes are Pod-lifetime, not container-lifetime, so -// every marker includes the writer PID and its Linux /proc start time. -func rolloutStateReadCommand(statePath string) string { - return fmt.Sprintf(`read -r state pid started extra < %s || exit 1; [ -z "$extra" ] || exit 1; case "$pid:$started" in *[!0-9:]*|:*|*:) exit 1;; esac; procstat="$(cat /proc/$pid/stat 2>/dev/null)" || exit 1; procstat="${procstat##*) }"; set -- $procstat; [ "$#" -ge 20 ] && [ "${20}" = "$started" ] || exit 1; `, statePath) -} - func hasRolloutMode(annotations map[string]string) bool { return strings.EqualFold(annotations[preparedRolloutModeAnnotation], preparedRolloutModeV1) } diff --git a/internal/controller/datadogagentinternal/prepared_rollout_support.go b/internal/controller/datadogagentinternal/prepared_rollout_support.go index 7abd185695..71c27aa03b 100644 --- a/internal/controller/datadogagentinternal/prepared_rollout_support.go +++ b/internal/controller/datadogagentinternal/prepared_rollout_support.go @@ -148,7 +148,7 @@ func daemonSetControlledByDDAI(ds *appsv1.DaemonSet, ddai *datadoghqv1alpha1.Dat } func preparedRolloutDaemonSetEligible(ds *appsv1.DaemonSet) bool { - if ds.DeletionTimestamp != nil || ds.Status.DesiredNumberScheduled <= 0 || ds.Status.ObservedGeneration != ds.Generation { + if ds.DeletionTimestamp != nil { return false } return ds.Spec.UpdateStrategy.Type == appsv1.RollingUpdateDaemonSetStrategyType && @@ -160,6 +160,7 @@ func currentDaemonSetRevision(ctx context.Context, reader client.Reader, ds *app if err := reader.List(ctx, revisions, client.InNamespace(ds.Namespace)); err != nil { return "", fmt.Errorf("list revisions for Agent DaemonSet %s/%s: %w", ds.Namespace, ds.Name, err) } + var current *appsv1.ControllerRevision for i := range revisions.Items { revision := &revisions.Items[i] diff --git a/internal/controller/datadogagentinternal/prepared_rollout_test.go b/internal/controller/datadogagentinternal/prepared_rollout_test.go index 2777e0902a..509dc404eb 100644 --- a/internal/controller/datadogagentinternal/prepared_rollout_test.go +++ b/internal/controller/datadogagentinternal/prepared_rollout_test.go @@ -5,7 +5,6 @@ package datadogagentinternal import ( - "strings" "testing" appsv1 "k8s.io/api/apps/v1" @@ -31,151 +30,121 @@ func TestPreparedRolloutRequiresExplicitMode(t *testing.T) { assert.True(t, preparedRolloutEnabled(ddai)) } -func TestPrepareAgentTemplateNetworkingMatrix(t *testing.T) { - tests := []struct { - name string - hostNetwork bool - hostPort int32 - wantError string - wantPortCount int - }{ - { - name: "pod network and UDS preserves declared container ports", - wantPortCount: 1, - }, - { - name: "pod network and hostPort is rejected", - hostPort: 8126, - wantError: "cannot overlap Pod-networked containers that declare hostPort", - }, - { - name: "host network strips scheduling port claims", - hostNetwork: true, - hostPort: 8126, - wantPortCount: 0, - }, +func TestPrepareAgentTemplatePreservesHostNetworkAndUDS(t *testing.T) { + ds := preparedTestDaemonSet(true) + require.NoError(t, prepareAgentTemplate(ds)) + assert.True(t, ds.Spec.Template.Spec.HostNetwork) + for i := range ds.Spec.Template.Spec.Containers { + assert.Empty(t, ds.Spec.Template.Spec.Containers[i].Ports) } + assert.True(t, hasHostPath(ds.Spec.Template.Spec.Volumes, "/var/run/datadog")) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ds := preparedTestDaemonSet(tt.hostNetwork) - ds.Spec.Template.Spec.Containers[1].Ports[0].HostPort = tt.hostPort - err := prepareAgentTemplate(ds) - if tt.wantError != "" { - require.ErrorContains(t, err, tt.wantError) - return - } - require.NoError(t, err) - for i := range ds.Spec.Template.Spec.Containers { - assert.Len(t, ds.Spec.Template.Spec.Containers[i].Ports, tt.wantPortCount) - } - // UDS hostPath volumes are deliberately preserved; sleeping processes - // do not bind or unlink the shared pathname. - assert.True(t, hasHostPath(ds.Spec.Template.Spec.Volumes, "/var/run/datadog")) - }) + podNetwork := preparedTestDaemonSet(false) + for i := range podNetwork.Spec.Template.Spec.Containers { + for j := range podNetwork.Spec.Template.Spec.Containers[i].Ports { + podNetwork.Spec.Template.Spec.Containers[i].Ports[j].HostPort = 0 + } } + require.NoError(t, prepareAgentTemplate(podNetwork)) + assert.False(t, podNetwork.Spec.Template.Spec.HostNetwork) + for i := range podNetwork.Spec.Template.Spec.Containers { + assert.NotEmpty(t, podNetwork.Spec.Template.Spec.Containers[i].Ports) + } + + podNetworkWithHostPort := preparedTestDaemonSet(false) + require.ErrorContains(t, prepareAgentTemplate(podNetworkWithHostPort), "declare hostPort") } -func TestConfigurePreparedRolloutUsesExistingBudget(t *testing.T) { - ddai := &datadoghqv1alpha1.DatadogAgentInternal{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ - preparedRolloutModeAnnotation: preparedRolloutModeV1, - }}} - ds := preparedTestDaemonSet(false) - ds.Spec.UpdateStrategy.RollingUpdate.MaxSurge = nil +func TestConfigurePreparedRolloutUsesExistingUnavailableBudgetAsSurge(t *testing.T) { + ddai := preparedRolloutDDAI() + ds := preparedTestDaemonSet(true) budget := intstr.FromString("10%") migrating, err := configurePreparedRollout(ddai, ds, nil, budget) require.NoError(t, err) assert.False(t, migrating) - require.NotNil(t, ds.Spec.UpdateStrategy.RollingUpdate) - assert.Equal(t, intstr.FromString("10%"), *ds.Spec.UpdateStrategy.RollingUpdate.MaxSurge) + assert.Equal(t, budget, *ds.Spec.UpdateStrategy.RollingUpdate.MaxSurge) assert.Equal(t, intstr.FromInt(0), *ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable) assert.Equal(t, preparedRolloutModeV1, ds.Spec.Template.Annotations[preparedRolloutModeAnnotation]) } func TestPreparedRolloutMigratesExistingProfileAntiAffinityBeforeSurge(t *testing.T) { - ddai := &datadoghqv1alpha1.DatadogAgentInternal{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ - preparedRolloutModeAnnotation: preparedRolloutModeV1, - }}} budget := intstr.FromInt(1) current := preparedTestDaemonSet(true) current.Generation = 1 current.Spec.Template.Spec.Affinity = &corev1.Affinity{PodAntiAffinity: broadAgentPodAntiAffinity()} desired := preparedTestDaemonSet(true) - migrating, err := configurePreparedRollout(ddai, desired, current, budget) + migrating, err := configurePreparedRollout(preparedRolloutDDAI(), desired, current, budget) require.NoError(t, err) require.True(t, migrating) assert.Equal(t, intstr.FromInt(0), *desired.Spec.UpdateStrategy.RollingUpdate.MaxSurge) assert.Equal(t, budget, *desired.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable) - assert.False(t, hasRolloutMode(desired.Spec.Template.Annotations)) - expectedAffinity, ok := profileSurgePodAntiAffinity(desired.Spec.Template.Labels) - require.True(t, ok) - assert.Equal(t, expectedAffinity, desired.Spec.Template.Spec.Affinity.PodAntiAffinity) assert.Nil(t, containerEnv(&desired.Spec.Template.Spec.Containers[0], rolloutEnabledEnv)) current = desired.DeepCopy() current.Generation = 2 - current.Status = appsv1.DaemonSetStatus{ObservedGeneration: 1, DesiredNumberScheduled: 2, UpdatedNumberScheduled: 1, NumberAvailable: 1, NumberUnavailable: 1} - desired = preparedTestDaemonSet(true) - migrating, err = configurePreparedRollout(ddai, desired, current, budget) - require.NoError(t, err) - require.True(t, migrating, "surge must wait until every old broad-affinity Pod is replaced") - current.Status = appsv1.DaemonSetStatus{ObservedGeneration: 2, DesiredNumberScheduled: 2, UpdatedNumberScheduled: 2, NumberAvailable: 2} desired = preparedTestDaemonSet(true) - migrating, err = configurePreparedRollout(ddai, desired, current, budget) + migrating, err = configurePreparedRollout(preparedRolloutDDAI(), desired, current, budget) require.NoError(t, err) assert.False(t, migrating) assert.Equal(t, budget, *desired.Spec.UpdateStrategy.RollingUpdate.MaxSurge) - assert.True(t, hasRolloutMode(desired.Spec.Template.Annotations)) } -func TestPreparedReplacementReportsPreparedAsReady(t *testing.T) { - ds := preparedTestDaemonSet(false) +func TestPreparedRolloutPreservesProbes(t *testing.T) { + ds := preparedTestDaemonSet(true) + original := make(map[string]struct { + startup *corev1.Probe + liveness *corev1.Probe + readiness *corev1.Probe + }, len(ds.Spec.Template.Spec.Containers)) + for i := range ds.Spec.Template.Spec.Containers { + container := &ds.Spec.Template.Spec.Containers[i] + original[container.Name] = struct { + startup *corev1.Probe + liveness *corev1.Probe + readiness *corev1.Probe + }{container.StartupProbe.DeepCopy(), container.LivenessProbe.DeepCopy(), container.ReadinessProbe.DeepCopy()} + } require.NoError(t, prepareAgentTemplate(ds)) for i := range ds.Spec.Template.Spec.Containers { container := &ds.Spec.Template.Spec.Containers[i] - require.NotNil(t, container.ReadinessProbe) - command := strings.Join(container.ReadinessProbe.Exec.Command, " ") - assert.Contains(t, command, "prepared) exit 0") - assert.NotContains(t, command, "prepared|activating") - assert.Contains(t, command, `/proc/$pid/stat`) - assert.Contains(t, command, `${20}`) - require.NotNil(t, container.LivenessProbe) - assert.NotContains(t, strings.Join(container.LivenessProbe.Exec.Command, " "), "prepared|activating") - assert.Contains(t, strings.Join(container.LivenessProbe.Exec.Command, " "), "prepared) exit 0") - - uidEnv := containerEnv(container, rolloutPodUIDEnv) - require.NotNil(t, uidEnv) - require.NotNil(t, uidEnv.ValueFrom) - require.NotNil(t, uidEnv.ValueFrom.FieldRef) - assert.Equal(t, "metadata.uid", uidEnv.ValueFrom.FieldRef.FieldPath) - stateEnv := containerEnv(container, rolloutStatePathEnv) - require.NotNil(t, stateEnv) - assert.True(t, strings.HasPrefix(stateEnv.Value, preparedRolloutStateDir+"/")) + assert.Equal(t, original[container.Name].startup, container.StartupProbe) + assert.Equal(t, original[container.Name].liveness, container.LivenessProbe) + assert.Equal(t, original[container.Name].readiness, container.ReadinessProbe) + assert.Equal(t, "metadata.uid", containerEnv(container, rolloutPodUIDEnv).ValueFrom.FieldRef.FieldPath) + assert.Equal(t, "status.hostIP", containerEnv(container, kubeletHostEnv).ValueFrom.FieldRef.FieldPath) } } -func TestPreparedRolloutStateUsesReservedPrivateEmptyDir(t *testing.T) { - ds := preparedTestDaemonSet(false) - require.NoError(t, prepareAgentTemplate(ds)) - volume := findVolumeByName(ds.Spec.Template.Spec.Volumes, preparedRolloutStateVolume) - require.NotNil(t, volume) - require.NotNil(t, volume.EmptyDir) - for i := range ds.Spec.Template.Spec.Containers { - assert.Contains(t, ds.Spec.Template.Spec.Containers[i].VolumeMounts, corev1.VolumeMount{Name: preparedRolloutStateVolume, MountPath: preparedRolloutStateDir}) +func TestPreparedRolloutSupportsAllRenderedAgentContainers(t *testing.T) { + ds := preparedTestDaemonSet(true) + ds.Spec.Template.Spec.Containers = []corev1.Container{ + {Name: string(apicommon.CoreAgentContainerName), Command: []string{"agent", "run"}}, + {Name: string(apicommon.TraceAgentContainerName), Command: []string{"/entrypoint.sh", "trace-agent"}}, + {Name: string(apicommon.ProcessAgentContainerName), Command: []string{"process-agent"}}, + {Name: string(apicommon.SystemProbeContainerName), Command: []string{"system-probe"}}, + {Name: string(apicommon.HostProfiler), Command: []string{"host-profiler", "--core-config=/etc/datadog-agent/datadog.yaml"}}, + {Name: string(apicommon.OtelAgent), Command: []string{"otel-agent"}}, + {Name: string(apicommon.PrivateActionRunnerContainerName), Command: []string{"/opt/datadog-agent/embedded/bin/privateactionrunner"}}, } + ds.Spec.Template.Spec.InitContainers = append(ds.Spec.Template.Spec.InitContainers, + corev1.Container{Name: "seccomp-setup"}, corev1.Container{Name: "host-profiler-seccomp-setup"}) + require.NoError(t, prepareAgentTemplate(ds)) + assert.Equal(t, []string{"trace-agent"}, ds.Spec.Template.Spec.Containers[1].Command) +} - conflicting := preparedTestDaemonSet(false) - conflicting.Spec.Template.Spec.Volumes = append(conflicting.Spec.Template.Spec.Volumes, corev1.Volume{Name: "shared", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run"}}}) - conflicting.Spec.Template.Spec.Containers[0].VolumeMounts = append(conflicting.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{Name: "shared", MountPath: "/var/run"}) - require.ErrorContains(t, prepareAgentTemplate(conflicting), "reserved name or path") +func TestPreparedRolloutSupportsStandaloneHostProfiler(t *testing.T) { + ds := preparedTestDaemonSet(true) + ds.Spec.Template.Spec.Containers = append(ds.Spec.Template.Spec.Containers, + corev1.Container{Name: string(apicommon.HostProfiler), Command: []string{"host-profiler"}}) + require.NoError(t, prepareAgentTemplate(ds)) } -func TestPreparedRolloutRejectsUngatedComponents(t *testing.T) { - ds := preparedTestDaemonSet(false) - ds.Spec.Template.Spec.Containers = append(ds.Spec.Template.Spec.Containers, corev1.Container{Name: "system-probe"}) - require.ErrorContains(t, prepareAgentTemplate(ds), "supports exactly agent and trace-agent") +func TestPreparedRolloutRejectsUnknownSidecar(t *testing.T) { + ds := preparedTestDaemonSet(true) + ds.Spec.Template.Spec.Containers = append(ds.Spec.Template.Spec.Containers, corev1.Container{Name: "unknown-sidecar", Command: []string{"sidecar"}}) + require.ErrorContains(t, prepareAgentTemplate(ds), "does not support container") } func TestProfileSurgeAntiAffinityAllowsOnlySameProfileOverlap(t *testing.T) { @@ -184,8 +153,6 @@ func TestProfileSurgeAntiAffinityAllowsOnlySameProfileOverlap(t *testing.T) { constants.ProfileLabelKey: "blue", }) require.True(t, ok) - require.Len(t, antiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, 2) - blocked := func(podLabels map[string]string) bool { for _, term := range antiAffinity.RequiredDuringSchedulingIgnoredDuringExecution { selector, err := metav1.LabelSelectorAsSelector(term.LabelSelector) @@ -199,11 +166,16 @@ func TestProfileSurgeAntiAffinityAllowsOnlySameProfileOverlap(t *testing.T) { base := map[string]string{apicommon.AgentDeploymentComponentLabelKey: constants.DefaultAgentResourceSuffix} assert.False(t, blocked(mergeLabels(base, map[string]string{apicommon.AgentDeploymentNameLabelKey: "agent-a", constants.ProfileLabelKey: "blue"}))) assert.True(t, blocked(mergeLabels(base, map[string]string{apicommon.AgentDeploymentNameLabelKey: "agent-a", constants.ProfileLabelKey: "green"}))) - assert.True(t, blocked(mergeLabels(base, map[string]string{apicommon.AgentDeploymentNameLabelKey: "agent-b", constants.ProfileLabelKey: "blue"}))) +} + +func preparedRolloutDDAI() *datadoghqv1alpha1.DatadogAgentInternal { + return &datadoghqv1alpha1.DatadogAgentInternal{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ + preparedRolloutModeAnnotation: preparedRolloutModeV1, + }}} } func preparedTestDaemonSet(hostNetwork bool) *appsv1.DaemonSet { - port := corev1.ContainerPort{Name: "declared", ContainerPort: 8126, Protocol: corev1.ProtocolTCP} + port := corev1.ContainerPort{Name: "declared", ContainerPort: 8126, HostPort: 8126, Protocol: corev1.ProtocolTCP} return &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{Name: "agent", Namespace: "default"}, Spec: appsv1.DaemonSetSpec{ @@ -222,25 +194,13 @@ func preparedTestDaemonSet(hostNetwork bool) *appsv1.DaemonSet { {Name: string(apicommon.InitConfigContainerName)}, }, Containers: []corev1.Container{ - {Name: string(apicommon.CoreAgentContainerName), Command: []string{"agent", "run"}, Ports: []corev1.ContainerPort{port}, LivenessProbe: &corev1.Probe{}}, - {Name: string(apicommon.TraceAgentContainerName), Command: []string{"/entrypoint.sh", "trace-agent"}, Ports: []corev1.ContainerPort{port}, LivenessProbe: &corev1.Probe{}}, + {Name: string(apicommon.CoreAgentContainerName), Command: []string{"agent", "run"}, Ports: []corev1.ContainerPort{port}, StartupProbe: &corev1.Probe{}, LivenessProbe: &corev1.Probe{}, ReadinessProbe: &corev1.Probe{}}, + {Name: string(apicommon.TraceAgentContainerName), Command: []string{"/entrypoint.sh", "trace-agent"}, Ports: []corev1.ContainerPort{port}, StartupProbe: &corev1.Probe{}, LivenessProbe: &corev1.Probe{}, ReadinessProbe: &corev1.Probe{}}, }, - Volumes: []corev1.Volume{{ - Name: "sockets", - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/run/datadog", - Type: ptr.To(corev1.HostPathDirectoryOrCreate), - }}, - }}, - }, - }, - UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ - Type: appsv1.RollingUpdateDaemonSetStrategyType, - RollingUpdate: &appsv1.RollingUpdateDaemonSet{ - MaxUnavailable: ptr.To(intstr.FromInt(1)), - MaxSurge: ptr.To(intstr.FromInt(1)), + Volumes: []corev1.Volume{{Name: "sockets", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/datadog", Type: ptr.To(corev1.HostPathDirectoryOrCreate)}}}}, }, }, + UpdateStrategy: appsv1.DaemonSetUpdateStrategy{Type: appsv1.RollingUpdateDaemonSetStrategyType, RollingUpdate: &appsv1.RollingUpdateDaemonSet{MaxUnavailable: ptr.To(intstr.FromInt(1)), MaxSurge: ptr.To(intstr.FromInt(1))}}, }, } } @@ -254,15 +214,6 @@ func hasHostPath(volumes []corev1.Volume, path string) bool { return false } -func findVolumeByName(volumes []corev1.Volume, name string) *corev1.Volume { - for i := range volumes { - if volumes[i].Name == name { - return &volumes[i] - } - } - return nil -} - func containerEnv(container *corev1.Container, name string) *corev1.EnvVar { for i := range container.Env { if container.Env[i].Name == name { diff --git a/internal/controller/datadogagentinternal/resource_fallback.go b/internal/controller/datadogagentinternal/resource_fallback.go index b6b13c62c3..8395a09c95 100644 --- a/internal/controller/datadogagentinternal/resource_fallback.go +++ b/internal/controller/datadogagentinternal/resource_fallback.go @@ -7,7 +7,6 @@ package datadogagentinternal import ( "context" "fmt" - "slices" "sort" "strconv" "strings" @@ -15,14 +14,9 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/util/intstr" - resourcehelper "k8s.io/component-helpers/resource" - "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -30,26 +24,24 @@ import ( datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" ) -const resourceFallbackOldPodAnnotation = "agent.datadoghq.com/resource-fallback-old-pod-uid" - -type resourceShortage struct { - cpu bool - memory bool -} - type fallbackCandidate struct { pending *corev1.Pod old *corev1.Pod nodeName string - shortage resourceShortage - reserved bool } -// reconcileResourceFallback breaks the maxSurge capacity deadlock only when -// the scheduler reports CPU and/or memory as the sole target-node blocker and -// removing the exact old Agent Pod is sufficient to make the replacement fit. -// The original maxUnavailable value is reused as the deletion budget. -func (r *Reconciler) reconcileResourceFallback(ctx context.Context, ddai *datadoghqv1alpha1.DatadogAgentInternal, expectedDS *appsv1.DaemonSet, budgetValue intstr.IntOrString) (reconcile.Result, error) { +type runningHandoffCandidate struct { + replacement *corev1.Pod + old *corev1.Pod + nodeName string +} + +const resourceFallbackPollInterval = 5 * time.Second + +// reconcilePreparedRollout authorizes one node handoff when a surged replacement +// is running but intentionally unready. If no replacement fits on its node, it +// retains the CPU/memory-only delete-before-create fallback. +func (r *Reconciler) reconcilePreparedRollout(ctx context.Context, ddai *datadoghqv1alpha1.DatadogAgentInternal, expectedDS *appsv1.DaemonSet, budgetValue intstr.IntOrString) (reconcile.Result, error) { reader := r.apiReader if reader == nil { reader = r.client @@ -61,6 +53,9 @@ func (r *Reconciler) reconcileResourceFallback(ctx context.Context, ddai *datado if !daemonSetControlledByDDAI(ds, ddai) || !preparedRolloutDaemonSetEligible(ds) || !hasRolloutMode(ds.Spec.Template.Annotations) { return reconcile.Result{}, nil } + if ds.Status.DesiredNumberScheduled <= 0 || ds.Status.ObservedGeneration != ds.Generation { + return resourceFallbackPollResult(ds), nil + } budget, err := intstr.GetScaledValueFromIntOrPercent(&budgetValue, int(ds.Status.DesiredNumberScheduled), true) if err != nil { return reconcile.Result{}, fmt.Errorf("resolve Agent resource fallback budget: %w", err) @@ -68,78 +63,235 @@ func (r *Reconciler) reconcileResourceFallback(ctx context.Context, ddai *datado if budget <= 0 { return reconcile.Result{}, nil } - revision, err := currentDaemonSetRevision(ctx, reader, ds) - if err != nil || revision == "" { + pods, err := daemonSetPods(ctx, reader, ds) + if err != nil { return reconcile.Result{}, err } - pods, err := daemonSetPods(ctx, reader, ds) + if consumedPreparedRolloutBudget(ds, pods) >= budget { + return resourceFallbackPollResult(ds), nil + } + desiredRevision, err := currentDaemonSetRevision(ctx, reader, ds) if err != nil { return reconcile.Result{}, err } - consumed := consumedResourceFallbackBudget(ds, pods, revision, time.Now()) - if consumed > budget { - return reconcile.Result{}, nil + if desiredRevision == "" { + return resourceFallbackPollResult(ds), nil } - for _, candidate := range fallbackCandidates(ds, pods, revision, time.Now()) { - if !candidate.reserved && consumed >= budget { - break - } - if !candidate.reserved { - live, err := r.revalidateFallbackCandidate(ctx, reader, ds, candidate, revision, false) - if err != nil { - return reconcile.Result{}, err - } - if live == nil { - continue - } - candidate = *live - base := candidate.pending.DeepCopy() - patched := candidate.pending.DeepCopy() - if patched.Annotations == nil { - patched.Annotations = map[string]string{} - } - patched.Annotations[resourceFallbackOldPodAnnotation] = string(candidate.old.UID) - if err := r.client.Patch(ctx, patched, client.MergeFrom(base)); err != nil { - return reconcile.Result{}, fmt.Errorf("reserve Agent resource fallback for Pod %s/%s: %w", patched.Namespace, patched.Name, err) - } - candidate.pending = patched - candidate.reserved = true + runningCandidates := runningHandoffCandidates(ds, pods, desiredRevision, time.Now()) + if len(runningCandidates) > 0 { + candidate := runningCandidates[0] + replacement := &corev1.Pod{} + old := &corev1.Pod{} + if err := reader.Get(ctx, client.ObjectKeyFromObject(candidate.replacement), replacement); err != nil { + return reconcile.Result{}, client.IgnoreNotFound(err) } - - live, err := r.revalidateFallbackCandidate(ctx, reader, ds, candidate, revision, true) - if err != nil { - return reconcile.Result{}, err + if err := reader.Get(ctx, client.ObjectKeyFromObject(candidate.old), old); err != nil { + return reconcile.Result{}, client.IgnoreNotFound(err) } - if live == nil { - continue + if replacement.UID != candidate.replacement.UID || old.UID != candidate.old.UID || + !controlledByUID(replacement, ds.UID) || !controlledByUID(old, ds.UID) || + !replacementRunningForHandoff(replacement, ds, desiredRevision) || + old.Spec.NodeName != candidate.nodeName || replacement.Spec.NodeName != candidate.nodeName || + old.DeletionTimestamp != nil || !podAvailable(old, ds.Spec.MinReadySeconds, time.Now()) || + podRevision(old) == "" || podRevision(old) == desiredRevision { + return resourceFallbackPollResult(ds), nil } - withinBudget, err := resourceFallbackBudgetWithinLimit(ctx, reader, ds, budgetValue, revision) + allowed, err := preparedRolloutDeletionAllowed(ctx, reader, ds, budget, desiredRevision) if err != nil { return reconcile.Result{}, err } - if !withinBudget { - return reconcile.Result{RequeueAfter: time.Second}, nil + if !allowed { + return resourceFallbackPollResult(ds), nil } - uid := live.old.UID - if err := r.client.Delete(ctx, live.old, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { - return reconcile.Result{}, fmt.Errorf("delete old Agent Pod %s/%s for resource fallback: %w", live.old.Namespace, live.old.Name, err) + + uid := old.UID + if err := r.client.Delete(ctx, old, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("delete old Agent Pod %s/%s for prepared handoff: %w", old.Namespace, old.Name, err) } - ctrl.LoggerFrom(ctx).WithValues("daemonset", ds.Name, "node", live.nodeName, "oldPod", live.old.Name, "replacementPod", live.pending.Name).Info("Deleted old Agent Pod after proving the surged replacement was blocked only by node CPU or memory") + ctrl.LoggerFrom(ctx).WithValues("daemonset", ds.Name, "node", candidate.nodeName, "oldPod", old.Name, "replacementPod", replacement.Name).Info("Deleted old Agent Pod after every replacement container started") if r.recorder != nil { - r.recorder.Eventf(ddai, corev1.EventTypeWarning, "AgentResourceFallback", "Deleted old Agent Pod %s on node %s because replacement %s could not fit alongside it", live.old.Name, live.nodeName, live.pending.Name) + r.recorder.Eventf(ddai, corev1.EventTypeNormal, "AgentPreparedHandoff", "Deleted old Agent Pod %s on node %s after every container in replacement %s started", old.Name, candidate.nodeName, replacement.Name) } return reconcile.Result{RequeueAfter: time.Second}, nil } - return reconcile.Result{}, nil + + candidates := fallbackCandidates(ds, pods, desiredRevision, time.Now()) + if len(candidates) == 0 { + return resourceFallbackPollResult(ds), nil + } + candidate := candidates[0] + + // Re-read both Pods immediately before deletion. The scheduler condition, + // target node and old Pod identity must still describe the same handoff. + pending := &corev1.Pod{} + old := &corev1.Pod{} + if err := reader.Get(ctx, client.ObjectKeyFromObject(candidate.pending), pending); err != nil { + return reconcile.Result{}, client.IgnoreNotFound(err) + } + if err := reader.Get(ctx, client.ObjectKeyFromObject(candidate.old), old); err != nil { + return reconcile.Result{}, client.IgnoreNotFound(err) + } + if pending.UID != candidate.pending.UID || old.UID != candidate.old.UID || !controlledByUID(pending, ds.UID) || !controlledByUID(old, ds.UID) { + return resourceFallbackPollResult(ds), nil + } + if _, ok := resourceOnlyUnschedulable(pending); !ok || podRevision(pending) != desiredRevision || !replacementSpecMatchesTemplate(pending, ds) { + return resourceFallbackPollResult(ds), nil + } + nodeName, ok := targetNodeFromDaemonSetAffinity(pending) + if !ok || nodeName != candidate.nodeName || pending.Spec.NodeName != "" || pending.DeletionTimestamp != nil || old.Spec.NodeName != nodeName || old.DeletionTimestamp != nil || !podAvailable(old, ds.Spec.MinReadySeconds, time.Now()) || podRevision(old) == "" || podRevision(old) == desiredRevision { + return resourceFallbackPollResult(ds), nil + } + allowed, err := preparedRolloutDeletionAllowed(ctx, reader, ds, budget, desiredRevision) + if err != nil { + return reconcile.Result{}, err + } + if !allowed { + return resourceFallbackPollResult(ds), nil + } + + uid := old.UID + if err := r.client.Delete(ctx, old, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("delete old Agent Pod %s/%s for resource fallback: %w", old.Namespace, old.Name, err) + } + ctrl.LoggerFrom(ctx).WithValues("daemonset", ds.Name, "node", nodeName, "oldPod", old.Name, "replacementPod", pending.Name).Info("Deleted old Agent Pod because the surged replacement was blocked by node CPU or memory") + if r.recorder != nil { + r.recorder.Eventf(ddai, corev1.EventTypeWarning, "AgentResourceFallback", "Deleted old Agent Pod %s on node %s because replacement %s was blocked by CPU or memory", old.Name, nodeName, pending.Name) + } + return reconcile.Result{RequeueAfter: time.Second}, nil } -func fallbackCandidates(ds *appsv1.DaemonSet, pods []corev1.Pod, revision string, now time.Time) []fallbackCandidate { +// preparedRolloutDeletionAllowed narrows the unavoidable observation-to-delete +// race by rechecking the live rollout generation, revision and unavailability +// immediately before deleting an old Pod. +func preparedRolloutDeletionAllowed(ctx context.Context, reader client.Reader, expectedDS *appsv1.DaemonSet, budget int, desiredRevision string) (bool, error) { + ds := &appsv1.DaemonSet{} + if err := reader.Get(ctx, client.ObjectKeyFromObject(expectedDS), ds); err != nil { + return false, client.IgnoreNotFound(err) + } + if ds.UID != expectedDS.UID || ds.Generation != expectedDS.Generation || ds.Status.ObservedGeneration != ds.Generation || !preparedRolloutDaemonSetEligible(ds) || !hasRolloutMode(ds.Spec.Template.Annotations) { + return false, nil + } + liveRevision, err := currentDaemonSetRevision(ctx, reader, ds) + if err != nil { + return false, err + } + if liveRevision == "" || liveRevision != desiredRevision { + return false, nil + } + pods, err := daemonSetPods(ctx, reader, ds) + if err != nil { + return false, err + } + return consumedPreparedRolloutBudget(ds, pods) < budget, nil +} + +func runningHandoffCandidates(ds *appsv1.DaemonSet, pods []corev1.Pod, desiredRevision string, now time.Time) []runningHandoffCandidate { + newByNode := map[string][]*corev1.Pod{} + oldByNode := map[string][]*corev1.Pod{} + for i := range pods { + pod := &pods[i] + if replacementRunningForHandoff(pod, ds, desiredRevision) { + newByNode[pod.Spec.NodeName] = append(newByNode[pod.Spec.NodeName], pod) + continue + } + if pod.Spec.NodeName != "" && pod.DeletionTimestamp == nil && podRevision(pod) != "" && podRevision(pod) != desiredRevision && podAvailable(pod, ds.Spec.MinReadySeconds, now) { + oldByNode[pod.Spec.NodeName] = append(oldByNode[pod.Spec.NodeName], pod) + } + } + + var candidates []runningHandoffCandidate + for nodeName, replacements := range newByNode { + olds := oldByNode[nodeName] + if len(replacements) == 1 && len(olds) == 1 { + candidates = append(candidates, runningHandoffCandidate{replacement: replacements[0], old: olds[0], nodeName: nodeName}) + } + } + sort.Slice(candidates, func(i, j int) bool { return candidates[i].nodeName < candidates[j].nodeName }) + return candidates +} + +func replacementRunningForHandoff(pod *corev1.Pod, ds *appsv1.DaemonSet, desiredRevision string) bool { + if pod.DeletionTimestamp != nil || pod.Spec.NodeName == "" || pod.Status.Phase != corev1.PodRunning || podRevision(pod) != desiredRevision || !podInitialized(pod) { + return false + } + if !replacementSpecMatchesTemplate(pod, ds) { + return false + } + + desiredImages := make(map[string]string, len(ds.Spec.Template.Spec.Containers)) + for i := range ds.Spec.Template.Spec.Containers { + container := &ds.Spec.Template.Spec.Containers[i] + if container.Name == "" { + return false + } + desiredImages[container.Name] = container.Image + } + if len(pod.Status.ContainerStatuses) != len(desiredImages) { + return false + } + + seen := make(map[string]struct{}, len(pod.Status.ContainerStatuses)) + for i := range pod.Status.ContainerStatuses { + status := &pod.Status.ContainerStatuses[i] + if _, ok := desiredImages[status.Name]; !ok || status.State.Running == nil || status.RestartCount != 0 || status.ContainerID == "" || status.ImageID == "" { + return false + } + if _, duplicate := seen[status.Name]; duplicate { + return false + } + seen[status.Name] = struct{}{} + } + return len(seen) == len(desiredImages) +} + +func replacementSpecMatchesTemplate(pod *corev1.Pod, ds *appsv1.DaemonSet) bool { + desiredImages := make(map[string]string, len(ds.Spec.Template.Spec.Containers)) + for i := range ds.Spec.Template.Spec.Containers { + container := &ds.Spec.Template.Spec.Containers[i] + if container.Name == "" { + return false + } + desiredImages[container.Name] = container.Image + } + if len(pod.Spec.Containers) != len(desiredImages) { + return false + } + for i := range pod.Spec.Containers { + container := &pod.Spec.Containers[i] + if desiredImage, ok := desiredImages[container.Name]; !ok || desiredImage != container.Image { + return false + } + } + return true +} + +func podInitialized(pod *corev1.Pod) bool { + for i := range pod.Status.Conditions { + condition := &pod.Status.Conditions[i] + if condition.Type == corev1.PodInitialized { + return condition.Status == corev1.ConditionTrue + } + } + return false +} + +func podRevision(pod *corev1.Pod) string { + return pod.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] +} + +func resourceFallbackPollResult(ds *appsv1.DaemonSet) reconcile.Result { + if daemonSetFullyRolledOut(ds) { + return reconcile.Result{} + } + return reconcile.Result{RequeueAfter: resourceFallbackPollInterval} +} + +func fallbackCandidates(ds *appsv1.DaemonSet, pods []corev1.Pod, desiredRevision string, now time.Time) []fallbackCandidate { var candidates []fallbackCandidate for i := range pods { pending := &pods[i] - shortage, ok := resourceOnlyUnschedulable(pending) - if !ok || !resourceFallbackSchedulingShapeSafe(pending) || pending.DeletionTimestamp != nil || pending.Spec.NodeName != "" || pending.Status.NominatedNodeName != "" || pending.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] != revision { + if _, ok := resourceOnlyUnschedulable(pending); !ok || pending.DeletionTimestamp != nil || pending.Spec.NodeName != "" || pending.Status.NominatedNodeName != "" || podRevision(pending) != desiredRevision || !replacementSpecMatchesTemplate(pending, ds) { continue } nodeName, ok := targetNodeFromDaemonSetAffinity(pending) @@ -149,29 +301,37 @@ func fallbackCandidates(ds *appsv1.DaemonSet, pods []corev1.Pod, revision string var oldPods []*corev1.Pod for j := range pods { old := &pods[j] - oldRevision := old.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] - if old.Spec.NodeName == nodeName && oldRevision != "" && oldRevision != revision && podAvailable(old, ds.Spec.MinReadySeconds, now) { + if old.Spec.NodeName == nodeName && old.DeletionTimestamp == nil && podRevision(old) != "" && podRevision(old) != desiredRevision && podAvailable(old, ds.Spec.MinReadySeconds, now) { oldPods = append(oldPods, old) } } - if len(oldPods) != 1 { - continue - } - reservedUID := pending.Annotations[resourceFallbackOldPodAnnotation] - if reservedUID != "" && reservedUID != string(oldPods[0].UID) { - continue + if len(oldPods) == 1 { + candidates = append(candidates, fallbackCandidate{pending: pending, old: oldPods[0], nodeName: nodeName}) } - candidates = append(candidates, fallbackCandidate{pending: pending, old: oldPods[0], nodeName: nodeName, shortage: shortage, reserved: reservedUID != ""}) } - sort.Slice(candidates, func(i, j int) bool { - if candidates[i].reserved != candidates[j].reserved { - return candidates[i].reserved - } - return candidates[i].nodeName < candidates[j].nodeName - }) + sort.Slice(candidates, func(i, j int) bool { return candidates[i].nodeName < candidates[j].nodeName }) return candidates } +func consumedPreparedRolloutBudget(ds *appsv1.DaemonSet, pods []corev1.Pod) int { + terminatingNodes := map[string]struct{}{} + for i := range pods { + if pods[i].DeletionTimestamp != nil && pods[i].Spec.NodeName != "" { + terminatingNodes[pods[i].Spec.NodeName] = struct{}{} + } + } + // NumberUnavailable may already include some terminating Pods. Counting both + // is deliberately conservative: fallback must never delete more old Agents + // merely because DaemonSet status and Pod deletion are observed at different + // times. + return int(ds.Status.NumberUnavailable) + len(terminatingNodes) +} + +type resourceShortage struct { + cpu bool + memory bool +} + func resourceOnlyUnschedulable(pod *corev1.Pod) (resourceShortage, bool) { condition := scheduledCondition(pod) if condition == nil || condition.Status != corev1.ConditionFalse || condition.Reason != corev1.PodReasonUnschedulable { @@ -246,383 +406,3 @@ func targetNodeFromDaemonSetAffinity(pod *corev1.Pod) (string, bool) { } return target, target != "" } - -func resourceFallbackSchedulingShapeSafe(pod *corev1.Pod) bool { - if pod.Spec.SchedulerName != "" && pod.Spec.SchedulerName != corev1.DefaultSchedulerName || pod.Spec.RuntimeClassName != nil || len(pod.Spec.TopologySpreadConstraints) > 0 { - return false - } - if pod.Spec.Affinity != nil { - if pod.Spec.Affinity.PodAffinity != nil { - return false - } - if pod.Spec.Affinity.PodAntiAffinity != nil { - expected, ok := profileSurgePodAntiAffinity(pod.Labels) - if !ok || !apiequality.Semantic.DeepEqual(pod.Spec.Affinity.PodAntiAffinity, expected) { - return false - } - } - } - containers := append(append([]corev1.Container{}, pod.Spec.InitContainers...), pod.Spec.Containers...) - for i := range containers { - for _, port := range containers[i].Ports { - if port.HostPort != 0 { - return false - } - } - } - for i := range pod.Spec.Volumes { - source := pod.Spec.Volumes[i].VolumeSource - if source.EmptyDir == nil && source.HostPath == nil && source.ConfigMap == nil && source.Secret == nil && source.DownwardAPI == nil && source.Projected == nil { - return false - } - } - return true -} - -func profileSurgePodAntiAffinitySatisfied(pending *corev1.Pod, nodePods []corev1.Pod) (bool, error) { - if pending.Spec.Affinity == nil || pending.Spec.Affinity.PodAntiAffinity == nil { - return true, nil - } - for _, term := range pending.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution { - selector, err := metav1.LabelSelectorAsSelector(term.LabelSelector) - if err != nil { - return false, fmt.Errorf("parse prepared Agent Pod anti-affinity: %w", err) - } - for i := range nodePods { - pod := &nodePods[i] - if pod.Namespace == pending.Namespace && selector.Matches(labels.Set(pod.Labels)) { - return false, nil - } - } - } - return true, nil -} - -// existingPodsAllowPendingByRequiredAntiAffinity checks the symmetric half of -// inter-pod anti-affinity: a scheduled Pod can reject the pending replacement. -func existingPodsAllowPendingByRequiredAntiAffinity(pending *corev1.Pod, existingPods []corev1.Pod, targetNodeName string) (bool, error) { - for i := range existingPods { - existing := &existingPods[i] - if existing.Spec.NodeName == "" || existing.Spec.Affinity == nil || existing.Spec.Affinity.PodAntiAffinity == nil { - continue - } - for _, term := range existing.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution { - selector, err := podAffinityTermSelector(&term, existing.Labels) - if err != nil { - return false, fmt.Errorf("parse existing Pod %s/%s anti-affinity: %w", existing.Namespace, existing.Name, err) - } - if !selector.Matches(labels.Set(pending.Labels)) || !affinityTermMaySelectNamespace(&term, existing.Namespace, pending.Namespace) { - continue - } - if term.TopologyKey != corev1.LabelHostname || existing.Spec.NodeName == targetNodeName { - return false, nil - } - } - } - return true, nil -} - -func podAffinityTermSelector(term *corev1.PodAffinityTerm, sourceLabels map[string]string) (labels.Selector, error) { - selector, err := metav1.LabelSelectorAsSelector(term.LabelSelector) - if err != nil { - return nil, err - } - for _, key := range term.MatchLabelKeys { - if value, ok := sourceLabels[key]; ok { - requirement, reqErr := labels.NewRequirement(key, selection.In, []string{value}) - if reqErr != nil { - return nil, reqErr - } - selector = selector.Add(*requirement) - } - } - for _, key := range term.MismatchLabelKeys { - if value, ok := sourceLabels[key]; ok { - requirement, reqErr := labels.NewRequirement(key, selection.NotIn, []string{value}) - if reqErr != nil { - return nil, reqErr - } - selector = selector.Add(*requirement) - } - } - return selector, nil -} - -func affinityTermMaySelectNamespace(term *corev1.PodAffinityTerm, sourceNamespace, targetNamespace string) bool { - if slices.Contains(term.Namespaces, targetNamespace) || term.NamespaceSelector != nil { - return true - } - return len(term.Namespaces) == 0 && sourceNamespace == targetNamespace -} - -func (r *Reconciler) revalidateFallbackCandidate(ctx context.Context, reader client.Reader, expectedDS *appsv1.DaemonSet, candidate fallbackCandidate, revision string, requireReservation bool) (*fallbackCandidate, error) { - liveDS := &appsv1.DaemonSet{} - if getErr := reader.Get(ctx, client.ObjectKeyFromObject(expectedDS), liveDS); getErr != nil { - return nil, client.IgnoreNotFound(getErr) - } - if liveDS.UID != expectedDS.UID || liveDS.Generation != expectedDS.Generation || !preparedRolloutDaemonSetEligible(liveDS) || !hasRolloutMode(liveDS.Spec.Template.Annotations) { - return nil, nil - } - liveRevision, err := currentDaemonSetRevision(ctx, reader, liveDS) - if err != nil || liveRevision != revision { - return nil, err - } - pending := &corev1.Pod{} - old := &corev1.Pod{} - if getErr := reader.Get(ctx, client.ObjectKeyFromObject(candidate.pending), pending); getErr != nil { - return nil, client.IgnoreNotFound(getErr) - } - if getErr := reader.Get(ctx, client.ObjectKeyFromObject(candidate.old), old); getErr != nil { - return nil, client.IgnoreNotFound(getErr) - } - if pending.UID != candidate.pending.UID || old.UID != candidate.old.UID || !controlledByUID(pending, liveDS.UID) || !controlledByUID(old, liveDS.UID) { - return nil, nil - } - shortage, ok := resourceOnlyUnschedulable(pending) - nodeName, targetOK := targetNodeFromDaemonSetAffinity(pending) - reservation := pending.Annotations[resourceFallbackOldPodAnnotation] - if !ok || !resourceFallbackSchedulingShapeSafe(pending) || !targetOK || nodeName != candidate.nodeName || pending.Spec.NodeName != "" || pending.Status.NominatedNodeName != "" || pending.DeletionTimestamp != nil || pending.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] != revision || requireReservation && reservation != string(old.UID) || reservation != "" && reservation != string(old.UID) { - return nil, nil - } - if old.Spec.NodeName != nodeName || old.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] == "" || old.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] == revision || !podAvailable(old, liveDS.Spec.MinReadySeconds, time.Now()) { - return nil, nil - } - node := &corev1.Node{} - if getErr := reader.Get(ctx, client.ObjectKey{Name: nodeName}, node); getErr != nil { - return nil, client.IgnoreNotFound(getErr) - } - if !nodeReadyForResourceFallback(node) || !toleratesBlockingNodeTaints(pending.Spec.Tolerations, node.Spec.Taints) { - return nil, nil - } - matches, err := nodeaffinity.GetRequiredNodeAffinity(pending).Match(node) - if err != nil || !matches { - return nil, err - } - nodePods := &corev1.PodList{} - if listErr := reader.List(ctx, nodePods, client.MatchingFields{"spec.nodeName": nodeName}); listErr != nil { - return nil, fmt.Errorf("list Pods on node %s for Agent resource fallback: %w", nodeName, listErr) - } - affinitySatisfied, err := profileSurgePodAntiAffinitySatisfied(pending, nodePods.Items) - if err != nil || !affinitySatisfied { - return nil, err - } - clusterPods := &corev1.PodList{} - if listErr := reader.List(ctx, clusterPods); listErr != nil { - return nil, fmt.Errorf("list cluster Pods for Agent anti-affinity fallback safety: %w", listErr) - } - existingAffinitySatisfied, err := existingPodsAllowPendingByRequiredAntiAffinity(pending, clusterPods.Items, nodeName) - if err != nil || !existingAffinitySatisfied { - return nil, err - } - if !resourceFitAfterOldPodRemoval(node, nodePods.Items, pending, old, shortage) { - return nil, nil - } - return &fallbackCandidate{pending: pending, old: old, nodeName: nodeName, shortage: shortage, reserved: reservation != ""}, nil -} - -func resourceFallbackBudgetWithinLimit(ctx context.Context, reader client.Reader, expectedDS *appsv1.DaemonSet, budgetValue intstr.IntOrString, revision string) (bool, error) { - liveDS := &appsv1.DaemonSet{} - if err := reader.Get(ctx, client.ObjectKeyFromObject(expectedDS), liveDS); err != nil { - return false, client.IgnoreNotFound(err) - } - if liveDS.UID != expectedDS.UID || liveDS.Generation != expectedDS.Generation || !preparedRolloutDaemonSetEligible(liveDS) { - return false, nil - } - liveRevision, err := currentDaemonSetRevision(ctx, reader, liveDS) - if err != nil || liveRevision != revision { - return false, err - } - pods, err := daemonSetPods(ctx, reader, liveDS) - if err != nil { - return false, err - } - budget, err := intstr.GetScaledValueFromIntOrPercent(&budgetValue, int(liveDS.Status.DesiredNumberScheduled), true) - if err != nil || budget <= 0 { - return false, err - } - return consumedResourceFallbackBudget(liveDS, pods, revision, time.Now()) <= budget, nil -} - -func consumedResourceFallbackBudget(ds *appsv1.DaemonSet, pods []corev1.Pod, revision string, now time.Time) int { - availableByNode := map[string]bool{} - knownNodes := map[string]bool{} - for i := range pods { - pod := &pods[i] - nodeName := pod.Spec.NodeName - if nodeName == "" { - nodeName, _ = targetNodeFromDaemonSetAffinity(pod) - } - if nodeName == "" { - continue - } - knownNodes[nodeName] = true - if podAvailable(pod, ds.Spec.MinReadySeconds, now) { - availableByNode[nodeName] = true - } - } - - reservations := 0 - reservedUnavailable := 0 - for i := range pods { - pod := &pods[i] - if pod.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] != revision || pod.Annotations[resourceFallbackOldPodAnnotation] == "" || podAvailable(pod, ds.Spec.MinReadySeconds, now) { - continue - } - nodeName := pod.Spec.NodeName - if nodeName == "" { - nodeName, _ = targetNodeFromDaemonSetAffinity(pod) - } - reservations++ - if nodeName == "" || !availableByNode[nodeName] { - reservedUnavailable++ - } - } - - liveUnavailable := 0 - for nodeName := range knownNodes { - if !availableByNode[nodeName] { - liveUnavailable++ - } - } - if missing := int(ds.Status.DesiredNumberScheduled) - len(knownNodes); missing > 0 { - liveUnavailable += missing - } - statusBeyondLive := max(0, int(ds.Status.NumberUnavailable)-liveUnavailable) - return reservations + liveUnavailable - min(liveUnavailable, reservedUnavailable) + statusBeyondLive -} - -func toleratesBlockingNodeTaints(tolerations []corev1.Toleration, taints []corev1.Taint) bool { - for i := range taints { - taint := &taints[i] - if taint.Effect != corev1.TaintEffectNoSchedule && taint.Effect != corev1.TaintEffectNoExecute { - continue - } - tolerated := false - for j := range tolerations { - toleration := &tolerations[j] - if toleration.Effect != "" && toleration.Effect != taint.Effect { - continue - } - operator := toleration.Operator - if operator == "" { - operator = corev1.TolerationOpEqual - } - if operator == corev1.TolerationOpExists && (toleration.Key == "" || toleration.Key == taint.Key) || operator == corev1.TolerationOpEqual && toleration.Key == taint.Key && toleration.Value == taint.Value { - tolerated = true - break - } - } - if !tolerated { - return false - } - } - return true -} - -func nodeReadyForResourceFallback(node *corev1.Node) bool { - if node.Spec.Unschedulable || node.DeletionTimestamp != nil { - return false - } - ready := false - pressure := map[corev1.NodeConditionType]bool{ - corev1.NodeMemoryPressure: false, - corev1.NodeDiskPressure: false, - corev1.NodePIDPressure: false, - } - for i := range node.Status.Conditions { - condition := node.Status.Conditions[i] - switch condition.Type { - case corev1.NodeReady: - ready = condition.Status == corev1.ConditionTrue - case corev1.NodeMemoryPressure, corev1.NodeDiskPressure, corev1.NodePIDPressure: - if condition.Status != corev1.ConditionFalse { - return false - } - pressure[condition.Type] = true - case corev1.NodeNetworkUnavailable: - if condition.Status != corev1.ConditionFalse { - return false - } - } - } - return ready && pressure[corev1.NodeMemoryPressure] && pressure[corev1.NodeDiskPressure] && pressure[corev1.NodePIDPressure] -} - -func resourceFitAfterOldPodRemoval(node *corev1.Node, nodePods []corev1.Pod, replacement, old *corev1.Pod, shortage resourceShortage) bool { - if len(replacement.Spec.ResourceClaims) > 0 { - return false - } - used := corev1.ResourceList{} - oldFound := false - for i := range nodePods { - pod := &nodePods[i] - if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { - continue - } - addResources(used, schedulerPodRequests(pod)) - oldFound = oldFound || pod.UID == old.UID - } - if !oldFound { - return false - } - before := copyResources(used) - addResources(before, schedulerPodRequests(replacement)) - after := copyResources(used) - subtractResources(after, schedulerPodRequests(old)) - addResources(after, schedulerPodRequests(replacement)) - for _, name := range []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory} { - reported := name == corev1.ResourceCPU && shortage.cpu || name == corev1.ResourceMemory && shortage.memory - oldRequest := schedulerPodRequests(old)[name] - if reported && (!resourceExceeds(before, node.Status.Allocatable, name) || oldRequest.Sign() <= 0) { - return false - } - } - return resourcesFit(after, node.Status.Allocatable) -} - -func schedulerPodRequests(pod *corev1.Pod) corev1.ResourceList { - return resourcehelper.PodRequests(pod, resourcehelper.PodResourcesOptions{UseStatusResources: true, InPlacePodLevelResourcesVerticalScalingEnabled: true}) -} - -func copyResources(resources corev1.ResourceList) corev1.ResourceList { - result := make(corev1.ResourceList, len(resources)) - for name, quantity := range resources { - result[name] = quantity.DeepCopy() - } - return result -} - -func addResources(target, values corev1.ResourceList) { - for name, value := range values { - quantity := target[name] - quantity.Add(value) - target[name] = quantity - } -} - -func subtractResources(target, values corev1.ResourceList) { - for name, value := range values { - quantity := target[name] - quantity.Sub(value) - target[name] = quantity - } -} - -func resourceExceeds(requests, allocatable corev1.ResourceList, name corev1.ResourceName) bool { - request := requests[name] - available := allocatable[name] - return request.Cmp(available) > 0 -} - -func resourcesFit(requests, allocatable corev1.ResourceList) bool { - for name, request := range requests { - if request.Sign() <= 0 { - continue - } - available, ok := allocatable[name] - if !ok || request.Cmp(available) > 0 { - return false - } - } - return true -} diff --git a/internal/controller/datadogagentinternal/resource_fallback_test.go b/internal/controller/datadogagentinternal/resource_fallback_test.go index 3bceee938f..d6129ca6a9 100644 --- a/internal/controller/datadogagentinternal/resource_fallback_test.go +++ b/internal/controller/datadogagentinternal/resource_fallback_test.go @@ -13,10 +13,8 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -27,49 +25,150 @@ import ( "github.com/stretchr/testify/require" ) -func TestResourceFallbackDeletesOnlyOldPodThatMakesReplacementFit(t *testing.T) { +func TestResourceFallbackDeletesOneOldPodForResourceOnlyFailure(t *testing.T) { fixture := newResourceFallbackFixture(t) - result, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + result, err := fixture.reconciler.reconcilePreparedRollout(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) require.NoError(t, err) assert.Equal(t, time.Second, result.RequeueAfter) err = fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{}) assert.True(t, apierrors.IsNotFound(err)) - require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.pending), fixture.pending)) - assert.Equal(t, string(fixture.old.UID), fixture.pending.Annotations[resourceFallbackOldPodAnnotation]) + assert.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.pending), &corev1.Pod{})) } -func TestResourceFallbackFailsClosedForOtherSchedulingBlockers(t *testing.T) { - for _, tt := range []struct { - name string - mutate func(*resourceFallbackFixture) - }{ - {name: "mixed scheduler reason", mutate: func(f *resourceFallbackFixture) { - f.pending.Status.Conditions[0].Message = "0/1 nodes are available: 1 node(s) didn't have free ports for the requested pod ports, 1 Insufficient cpu." - }}, - {name: "hostPort introduced by admission", mutate: func(f *resourceFallbackFixture) { - f.pending.Spec.Containers[0].Ports = []corev1.ContainerPort{{ContainerPort: 8126, HostPort: 8126}} - }}, - {name: "old requests are insufficient", mutate: func(f *resourceFallbackFixture) { - f.pending.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU] = resource.MustParse("2") +func TestResourceFallbackIgnoresOtherSchedulingFailures(t *testing.T) { + fixture := newResourceFallbackFixture(t) + fixture.pending.Status.Conditions[0].Message = "0/1 nodes are available: 1 node(s) didn't have free ports for the requested pod ports, 1 Insufficient cpu." + updatePodStatus(t, fixture.client, fixture.pending) + + result, err := fixture.reconciler.reconcilePreparedRollout(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.Equal(t, resourceFallbackPollInterval, result.RequeueAfter) + assert.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{})) +} + +func TestResourceFallbackRespectsExistingUnavailableBudget(t *testing.T) { + fixture := newResourceFallbackFixture(t) + fixture.ds.Status.NumberUnavailable = 1 + updateDaemonSetStatus(t, fixture.client, fixture.ds) + + result, err := fixture.reconciler.reconcilePreparedRollout(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.Equal(t, resourceFallbackPollInterval, result.RequeueAfter) + assert.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{})) +} + +func TestResourceFallbackRequiresAnOldRevisionOnTheTargetNode(t *testing.T) { + fixture := newResourceFallbackFixture(t) + fixture.old.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] = "new-hash" + require.NoError(t, fixture.client.Update(context.Background(), fixture.old)) + + result, err := fixture.reconciler.reconcilePreparedRollout(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.Equal(t, resourceFallbackPollInterval, result.RequeueAfter) + assert.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{})) +} + +func TestPreparedRolloutDeletionRechecksLiveBudgetAndGeneration(t *testing.T) { + fixture := newResourceFallbackFixture(t) + + allowed, err := preparedRolloutDeletionAllowed(context.Background(), fixture.client, fixture.ds, 1, "new-hash") + require.NoError(t, err) + assert.True(t, allowed) + + fixture.ds.Status.NumberUnavailable = 1 + updateDaemonSetStatus(t, fixture.client, fixture.ds) + allowed, err = preparedRolloutDeletionAllowed(context.Background(), fixture.client, fixture.ds, 1, "new-hash") + require.NoError(t, err) + assert.False(t, allowed) + + fixture.ds.Status.NumberUnavailable = 0 + fixture.ds.Status.ObservedGeneration-- + updateDaemonSetStatus(t, fixture.client, fixture.ds) + allowed, err = preparedRolloutDeletionAllowed(context.Background(), fixture.client, fixture.ds, 1, "new-hash") + require.NoError(t, err) + assert.False(t, allowed) +} + +func TestResourceFallbackStopsPollingAfterRollout(t *testing.T) { + fixture := newResourceFallbackFixture(t) + fixture.ds.Status.UpdatedNumberScheduled = 1 + updateDaemonSetStatus(t, fixture.client, fixture.ds) + fixture.pending.Status.Conditions[0].Message = "0/1 nodes are available: 1 node(s) had untolerated taint." + updatePodStatus(t, fixture.client, fixture.pending) + + result, err := fixture.reconciler.reconcilePreparedRollout(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.Zero(t, result.RequeueAfter) +} + +func TestPreparedRolloutPollsUntilDaemonSetStatusObservesTemplate(t *testing.T) { + fixture := newResourceFallbackFixture(t) + fixture.ds.Status.ObservedGeneration-- + updateDaemonSetStatus(t, fixture.client, fixture.ds) + + result, err := fixture.reconciler.reconcilePreparedRollout(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.Equal(t, resourceFallbackPollInterval, result.RequeueAfter) +} + +func TestConsumedFallbackBudgetCountsTerminatingNodes(t *testing.T) { + fixture := newResourceFallbackFixture(t) + fixture.ds.Status.NumberUnavailable = 1 + now := metav1.Now() + terminatingA := fixture.old.DeepCopy() + terminatingA.DeletionTimestamp = &now + terminatingB := fixture.old.DeepCopy() + terminatingB.Name = "old-b" + terminatingB.UID = "old-b-uid" + terminatingB.Spec.NodeName = "node-b" + terminatingB.DeletionTimestamp = &now + assert.Equal(t, 3, consumedPreparedRolloutBudget(fixture.ds, []corev1.Pod{*terminatingA, *terminatingB})) +} + +func TestPreparedRolloutDeletesOldPodAfterReplacementContainersAreRunning(t *testing.T) { + fixture := newResourceFallbackFixture(t) + replacement := fixture.pending.DeepCopy() + replacement.Spec.NodeName = "node-a" + replacement.Spec.Affinity = nil + replacement.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] = "new-hash" + replacement.Status = corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{{Type: corev1.PodInitialized, Status: corev1.ConditionTrue}}, + ContainerStatuses: []corev1.ContainerStatus{{ + Name: "agent", Image: "agent:new", ImageID: "sha256:new", ContainerID: "containerd://new", + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: metav1.Now()}}, }}, - } { - t.Run(tt.name, func(t *testing.T) { + } + require.NoError(t, fixture.client.Delete(context.Background(), fixture.pending)) + replacement.ResourceVersion = "" + require.NoError(t, fixture.client.Create(context.Background(), replacement)) + require.NoError(t, fixture.client.Status().Update(context.Background(), replacement)) + + result, err := fixture.reconciler.reconcilePreparedRollout(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) + require.NoError(t, err) + assert.Equal(t, time.Second, result.RequeueAfter) + err = fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{}) + assert.True(t, apierrors.IsNotFound(err)) +} + +func TestPreparedRolloutRequiresEveryReplacementContainerToBePristineAndRunning(t *testing.T) { + tests := map[string]func(*corev1.Pod){ + "not initialized": func(p *corev1.Pod) { p.Status.Conditions = nil }, + "container terminated": func(p *corev1.Pod) { + p.Status.ContainerStatuses[0].State = corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{}} + }, + "container restarted": func(p *corev1.Pod) { p.Status.ContainerStatuses[0].RestartCount = 1 }, + "missing container id": func(p *corev1.Pod) { p.Status.ContainerStatuses[0].ContainerID = "" }, + "missing image id": func(p *corev1.Pod) { p.Status.ContainerStatuses[0].ImageID = "" }, + "wrong image": func(p *corev1.Pod) { p.Spec.Containers[0].Image = "agent:wrong" }, + "wrong revision": func(p *corev1.Pod) { p.Labels[appsv1.DefaultDaemonSetUniqueLabelKey] = "old-hash" }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { fixture := newResourceFallbackFixture(t) - tt.mutate(&fixture) - if tt.name == "mixed scheduler reason" { - _, safe := resourceOnlyUnschedulable(fixture.pending) - require.False(t, safe) - } - desiredStatus := *fixture.pending.Status.DeepCopy() - require.NoError(t, fixture.client.Update(context.Background(), fixture.pending)) - livePending := &corev1.Pod{} - require.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.pending), livePending)) - livePending.Status = desiredStatus - require.NoError(t, fixture.client.Status().Update(context.Background(), livePending)) - fixture.pending = livePending - _, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) - require.NoError(t, err) - assert.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{})) + replacement := runningReplacement(fixture) + mutate(replacement) + require.False(t, replacementRunningForHandoff(replacement, fixture.ds, "new-hash")) }) } } @@ -88,41 +187,6 @@ func TestResourceOnlyUnschedulableParsing(t *testing.T) { assert.False(t, ok) } -func TestResourceFallbackRejectsExistingPodAntiAffinity(t *testing.T) { - fixture := newResourceFallbackFixture(t) - blocker := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "blocker", Namespace: fixture.pending.Namespace, Labels: map[string]string{"app": "other"}}, - Spec: corev1.PodSpec{ - NodeName: "node-a", - Affinity: &corev1.Affinity{PodAntiAffinity: &corev1.PodAntiAffinity{RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ - LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "agent"}}, - TopologyKey: corev1.LabelHostname, - }}}}, - Containers: []corev1.Container{{Name: "blocker"}}, - }, - } - require.NoError(t, fixture.client.Create(context.Background(), blocker)) - - _, err := fixture.reconciler.reconcileResourceFallback(context.Background(), fixture.ddai, fixture.ds, intstr.FromInt(1)) - require.NoError(t, err) - assert.NoError(t, fixture.client.Get(context.Background(), client.ObjectKeyFromObject(fixture.old), &corev1.Pod{})) -} - -func TestConsumedResourceFallbackBudgetCountsDisjointUnavailableNodes(t *testing.T) { - fixture := newResourceFallbackFixture(t) - fixture.ds.Status.DesiredNumberScheduled = 2 - fixture.ds.Status.NumberUnavailable = 1 - fixture.pending.Annotations = map[string]string{resourceFallbackOldPodAnnotation: string(fixture.old.UID)} - unavailable := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "unavailable", Namespace: "default", Labels: map[string]string{appsv1.DefaultDaemonSetUniqueLabelKey: "old"}}, - Spec: corev1.PodSpec{NodeName: "node-b"}, - Status: corev1.PodStatus{Phase: corev1.PodPending}, - } - - consumed := consumedResourceFallbackBudget(fixture.ds, []corev1.Pod{*fixture.old, *fixture.pending, unavailable}, "new", time.Now()) - assert.Equal(t, 2, consumed) -} - type resourceFallbackFixture struct { client client.Client reconciler *Reconciler @@ -146,67 +210,76 @@ func newResourceFallbackFixture(t *testing.T) resourceFallbackFixture { }}}, Spec: appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "agent"}}, - Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"app": "agent"}, Annotations: map[string]string{preparedRolloutModeAnnotation: preparedRolloutModeV1}, - }}, - UpdateStrategy: appsv1.DaemonSetUpdateStrategy{Type: appsv1.RollingUpdateDaemonSetStrategyType, RollingUpdate: &appsv1.RollingUpdateDaemonSet{ - MaxSurge: ptr.To(intstr.FromInt(1)), MaxUnavailable: ptr.To(intstr.FromInt(0)), - }}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "agent"}, Annotations: map[string]string{preparedRolloutModeAnnotation: preparedRolloutModeV1}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "agent", Image: "agent:new"}}}, + }, + UpdateStrategy: appsv1.DaemonSetUpdateStrategy{Type: appsv1.RollingUpdateDaemonSetStrategyType, RollingUpdate: &appsv1.RollingUpdateDaemonSet{MaxSurge: ptr.To(intstr.FromInt(1)), MaxUnavailable: ptr.To(intstr.FromInt(0))}}, }, Status: appsv1.DaemonSetStatus{ObservedGeneration: 2, DesiredNumberScheduled: 1, NumberReady: 1, NumberAvailable: 1}, } owner := metav1.OwnerReference{APIVersion: appsv1.SchemeGroupVersion.String(), Kind: "DaemonSet", Name: ds.Name, UID: ds.UID, Controller: ptr.To(true)} old := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "old", Namespace: "default", UID: "old-uid", Labels: map[string]string{"app": "agent", appsv1.DefaultDaemonSetUniqueLabelKey: "old"}, OwnerReferences: []metav1.OwnerReference{owner}}, - Spec: corev1.PodSpec{NodeName: "node-a", Containers: []corev1.Container{{Name: "agent", Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}}}}}, + ObjectMeta: metav1.ObjectMeta{Name: "old", Namespace: "default", UID: "old-uid", Labels: map[string]string{"app": "agent", appsv1.DefaultDaemonSetUniqueLabelKey: "old-hash"}, OwnerReferences: []metav1.OwnerReference{owner}}, + Spec: corev1.PodSpec{NodeName: "node-a", Containers: []corev1.Container{{Name: "agent", Image: "agent:old"}}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue, LastTransitionTime: metav1.NewTime(time.Now().Add(-time.Minute))}}}, } pending := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "new", Namespace: "default", UID: "new-uid", Labels: map[string]string{"app": "agent", appsv1.DefaultDaemonSetUniqueLabelKey: "new"}, OwnerReferences: []metav1.OwnerReference{owner}}, + ObjectMeta: metav1.ObjectMeta{Name: "new", Namespace: "default", UID: "new-uid", Labels: map[string]string{"app": "agent", appsv1.DefaultDaemonSetUniqueLabelKey: "new-hash"}, OwnerReferences: []metav1.OwnerReference{owner}}, Spec: corev1.PodSpec{ Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchFields: []corev1.NodeSelectorRequirement{{Key: metav1.ObjectNameField, Operator: corev1.NodeSelectorOpIn, Values: []string{"node-a"}}}}}}}}, - Containers: []corev1.Container{{Name: "agent", Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}}}}, + Containers: []corev1.Container{{Name: "agent", Image: "agent:new"}}, }, Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: corev1.PodReasonUnschedulable, Message: "0/1 nodes are available: 1 Insufficient cpu."}}}, } - node := &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{Name: "node-a"}, - Status: corev1.NodeStatus{ - Allocatable: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1500m"), corev1.ResourceMemory: resource.MustParse("1Gi"), corev1.ResourcePods: resource.MustParse("10")}, - Conditions: []corev1.NodeCondition{ - {Type: corev1.NodeReady, Status: corev1.ConditionTrue}, - {Type: corev1.NodeMemoryPressure, Status: corev1.ConditionFalse}, - {Type: corev1.NodeDiskPressure, Status: corev1.ConditionFalse}, - {Type: corev1.NodePIDPressure, Status: corev1.ConditionFalse}, - {Type: corev1.NodeNetworkUnavailable, Status: corev1.ConditionFalse}, - }, + revisionData, err := json.Marshal(struct { + Spec struct { + Template corev1.PodTemplateSpec `json:"template"` + } `json:"spec"` + }{Spec: struct { + Template corev1.PodTemplateSpec `json:"template"` + }{Template: ds.Spec.Template}}) + require.NoError(t, err) + revision := &appsv1.ControllerRevision{ + ObjectMeta: metav1.ObjectMeta{ + Name: "agent-new-hash", Namespace: ds.Namespace, + Labels: map[string]string{appsv1.DefaultDaemonSetUniqueLabelKey: "new-hash"}, + OwnerReferences: []metav1.OwnerReference{owner}, }, + Data: runtime.RawExtension{Raw: revisionData}, + Revision: 2, } - revision := controllerRevisionForFallbackTest(t, ds, "new") - c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&corev1.Pod{}, &appsv1.DaemonSet{}).WithObjects(ddai, ds, old, pending, node, revision).WithIndex(&corev1.Pod{}, "spec.nodeName", func(obj client.Object) []string { - pod := obj.(*corev1.Pod) - if pod.Spec.NodeName == "" { - return nil - } - return []string{pod.Spec.NodeName} - }).Build() + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&corev1.Pod{}, &appsv1.DaemonSet{}).WithObjects(ddai, ds, old, pending, revision).Build() return resourceFallbackFixture{client: c, reconciler: &Reconciler{client: c, apiReader: c}, ddai: ddai, ds: ds, old: old, pending: pending} } -func controllerRevisionForFallbackTest(t *testing.T, ds *appsv1.DaemonSet, hash string) *appsv1.ControllerRevision { - t.Helper() - template, err := json.Marshal(ds.Spec.Template) - require.NoError(t, err) - var patch map[string]any - require.NoError(t, json.Unmarshal(template, &patch)) - patch["$patch"] = "replace" - raw, err := json.Marshal(map[string]any{"spec": map[string]any{"template": patch}}) - require.NoError(t, err) - return &appsv1.ControllerRevision{ - ObjectMeta: metav1.ObjectMeta{Name: "agent-new", Namespace: ds.Namespace, UID: types.UID("revision-uid"), Labels: map[string]string{appsv1.DefaultDaemonSetUniqueLabelKey: hash}, OwnerReferences: []metav1.OwnerReference{{ - APIVersion: appsv1.SchemeGroupVersion.String(), Kind: "DaemonSet", Name: ds.Name, UID: ds.UID, Controller: ptr.To(true), - }}}, - Revision: 2, - Data: runtime.RawExtension{Raw: raw}, +func runningReplacement(fixture resourceFallbackFixture) *corev1.Pod { + pod := fixture.pending.DeepCopy() + pod.Spec.NodeName = "node-a" + pod.Spec.Affinity = nil + pod.Status = corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{{Type: corev1.PodInitialized, Status: corev1.ConditionTrue}}, + ContainerStatuses: []corev1.ContainerStatus{{ + Name: "agent", Image: "agent:new", ImageID: "sha256:new", ContainerID: "containerd://new", + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: metav1.Now()}}, + }}, } + return pod +} + +func updatePodStatus(t *testing.T, c client.Client, pod *corev1.Pod) { + t.Helper() + live := &corev1.Pod{} + require.NoError(t, c.Get(context.Background(), client.ObjectKeyFromObject(pod), live)) + live.Status = *pod.Status.DeepCopy() + require.NoError(t, c.Status().Update(context.Background(), live)) +} + +func updateDaemonSetStatus(t *testing.T, c client.Client, ds *appsv1.DaemonSet) { + t.Helper() + live := &appsv1.DaemonSet{} + require.NoError(t, c.Get(context.Background(), client.ObjectKeyFromObject(ds), live)) + live.Status = ds.Status + require.NoError(t, c.Status().Update(context.Background(), live)) } diff --git a/internal/controller/datadogagentinternal_controller.go b/internal/controller/datadogagentinternal_controller.go index fb373f7832..e2a6de7dbb 100644 --- a/internal/controller/datadogagentinternal_controller.go +++ b/internal/controller/datadogagentinternal_controller.go @@ -13,8 +13,6 @@ import ( corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" - apiequality "k8s.io/apimachinery/pkg/api/equality" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -27,18 +25,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - apicommon "github.com/DataDog/datadog-operator/api/datadoghq/common" "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" "github.com/DataDog/datadog-operator/internal/controller/datadogagent/object" "github.com/DataDog/datadog-operator/internal/controller/datadogagentinternal" - "github.com/DataDog/datadog-operator/pkg/constants" "github.com/DataDog/datadog-operator/pkg/controller/utils/datadog" "github.com/DataDog/datadog-operator/pkg/kubernetes" ) -const preparedRolloutModeAnnotationKey = "experimental.agent.datadoghq.com/node-agent-rollout-mode" - // DatadogAgentInternalReconciler reconciles a DatadogAgentInternal object. type DatadogAgentInternalReconciler struct { client.Client @@ -52,8 +46,8 @@ type DatadogAgentInternalReconciler struct { // +kubebuilder:rbac:groups=datadoghq.com,resources=datadogagentinternals,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=datadoghq.com,resources=datadogagentinternals/status,verbs=get;update;patch // +kubebuilder:rbac:groups=datadoghq.com,resources=datadogagentinternals/finalizers,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;patch;delete -// +kubebuilder:rbac:groups=apps,resources=controllerrevisions,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;delete +// +kubebuilder:rbac:groups=apps,resources=controllerrevisions,verbs=get;list // Reconcile loop for DatadogAgent. func (r *DatadogAgentInternalReconciler) Reconcile(ctx context.Context, ddai *v1alpha1.DatadogAgentInternal) (ctrl.Result, error) { @@ -85,12 +79,6 @@ func (r *DatadogAgentInternalReconciler) SetupWithManager(mgr ctrl.Manager, metr handlerEnqueue := handler.EnqueueRequestsFromMapFunc(enqueueIfOwnedByDatadogAgentInternal) builder.Watches(&rbacv1.ClusterRole{}, handlerEnqueue) builder.Watches(&rbacv1.ClusterRoleBinding{}, handlerEnqueue) - builder.Watches( - &corev1.Pod{}, - handler.EnqueueRequestsFromMapFunc(enqueueDatadogAgentInternalForPod(mgr.GetAPIReader())), - ctrlbuilder.WithPredicates(preparedRolloutPodPredicate()), - ) - if r.Options.ExtendedDaemonsetOptions.Enabled { builder = builder.Owns(&edsdatadoghqv1alpha1.ExtendedDaemonSet{}) } @@ -136,90 +124,6 @@ func datadogAgentInternalEventPredicate() predicate.Predicate { ) } -func enqueueDatadogAgentInternalForPod(reader client.Reader) handler.MapFunc { - return func(ctx context.Context, obj client.Object) []reconcile.Request { - pod, ok := obj.(*corev1.Pod) - if !ok || pod.Labels[apicommon.AgentDeploymentComponentLabelKey] != constants.DefaultAgentResourceSuffix { - return nil - } - podOwner := metav1.GetControllerOf(pod) - if podOwner == nil || podOwner.APIVersion != appsv1.SchemeGroupVersion.String() || podOwner.Kind != "DaemonSet" { - return nil - } - ds := &appsv1.DaemonSet{} - if err := reader.Get(ctx, client.ObjectKey{Namespace: pod.Namespace, Name: podOwner.Name}, ds); err != nil || ds.UID != podOwner.UID { - return nil - } - if ds.Spec.Template.Annotations[preparedRolloutModeAnnotationKey] != "prepared-surge-v1" { - return nil - } - ddaiOwner := metav1.GetControllerOf(ds) - if ddaiOwner == nil || ddaiOwner.APIVersion != datadoghqv1alpha1.GroupVersion.String() || ddaiOwner.Kind != "DatadogAgentInternal" { - return nil - } - return []reconcile.Request{{NamespacedName: client.ObjectKey{Namespace: ds.Namespace, Name: ddaiOwner.Name}}} - } -} - -func preparedRolloutPodPredicate() predicate.Predicate { - return predicate.Funcs{ - CreateFunc: func(e event.CreateEvent) bool { - _, ok := e.Object.(*corev1.Pod) - return ok - }, - UpdateFunc: func(e event.UpdateEvent) bool { - oldPod, oldOK := e.ObjectOld.(*corev1.Pod) - newPod, newOK := e.ObjectNew.(*corev1.Pod) - if !oldOK || !newOK { - return false - } - return podConditionChanged(oldPod, newPod, corev1.PodScheduled) || - podConditionChanged(oldPod, newPod, corev1.PodReady) || - containerRolloutStatusChanged(oldPod.Status.InitContainerStatuses, newPod.Status.InitContainerStatuses) || - containerRolloutStatusChanged(oldPod.Status.ContainerStatuses, newPod.Status.ContainerStatuses) - }, - DeleteFunc: func(event.DeleteEvent) bool { return true }, - GenericFunc: func(event.GenericEvent) bool { return false }, - } -} - -func containerRolloutStatusChanged(oldStatuses, newStatuses []corev1.ContainerStatus) bool { - if len(oldStatuses) != len(newStatuses) { - return true - } - oldByName := make(map[string]corev1.ContainerStatus, len(oldStatuses)) - for i := range oldStatuses { - oldByName[oldStatuses[i].Name] = oldStatuses[i] - } - for i := range newStatuses { - old, found := oldByName[newStatuses[i].Name] - if !found || !apiequality.Semantic.DeepEqual(old.Started, newStatuses[i].Started) || old.RestartCount != newStatuses[i].RestartCount || - (old.State.Running == nil) != (newStatuses[i].State.Running == nil) || - (old.State.Terminated == nil) != (newStatuses[i].State.Terminated == nil) { - return true - } - } - return false -} - -func podConditionChanged(oldPod, newPod *corev1.Pod, conditionType corev1.PodConditionType) bool { - oldCondition := podCondition(oldPod, conditionType) - newCondition := podCondition(newPod, conditionType) - if oldCondition == nil || newCondition == nil { - return oldCondition != newCondition - } - return oldCondition.Status != newCondition.Status || oldCondition.Reason != newCondition.Reason || oldCondition.Message != newCondition.Message -} - -func podCondition(pod *corev1.Pod, conditionType corev1.PodConditionType) *corev1.PodCondition { - for i := range pod.Status.Conditions { - if pod.Status.Conditions[i].Type == conditionType { - return &pod.Status.Conditions[i] - } - } - return nil -} - func enqueueIfOwnedByDatadogAgentInternal(ctx context.Context, obj client.Object) []reconcile.Request { labels := obj.GetLabels() diff --git a/internal/controller/datadogagentinternal_controller_test.go b/internal/controller/datadogagentinternal_controller_test.go index 81c5226935..b2d0efc321 100644 --- a/internal/controller/datadogagentinternal_controller_test.go +++ b/internal/controller/datadogagentinternal_controller_test.go @@ -8,239 +8,15 @@ import ( "context" "testing" - edsdatadoghqv1alpha1 "github.com/DataDog/extendeddaemonset/api/v1alpha1" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "k8s.io/utils/ptr" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - ctrlconfig "sigs.k8s.io/controller-runtime/pkg/config" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/manager" - apicommon "github.com/DataDog/datadog-operator/api/datadoghq/common" - datadoghqv1alpha1 "github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1" - componentagent "github.com/DataDog/datadog-operator/internal/controller/datadogagent/component/agent" - "github.com/DataDog/datadog-operator/internal/controller/datadogagentinternal" - "github.com/DataDog/datadog-operator/pkg/constants" - "github.com/DataDog/datadog-operator/pkg/controller/utils/datadog" + "github.com/DataDog/datadog-operator/internal/controller/datadogagent/object" "github.com/DataDog/datadog-operator/pkg/kubernetes" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -type noopMetricsForwardersManager struct{} - -func (noopMetricsForwardersManager) Register(client.Object) {} -func (noopMetricsForwardersManager) Unregister(client.Object) {} -func (noopMetricsForwardersManager) ProcessError(client.Object, error) {} -func (noopMetricsForwardersManager) ProcessEvent(client.Object, datadog.Event) {} -func (noopMetricsForwardersManager) SetEnabledFeatures(client.Object, []string) {} -func (noopMetricsForwardersManager) MetricsForwarderStatusForObj(client.Object) *datadog.ConditionCommon { - return nil -} - -func TestDatadogAgentInternalSetupWithManager(t *testing.T) { - tests := []struct { - name string - options datadogagentinternal.ReconcilerOptions - }{ - {name: "default"}, - { - name: "optional watches and metrics", - options: datadogagentinternal.ReconcilerOptions{ - ExtendedDaemonsetOptions: componentagent.ExtendedDaemonsetOptions{Enabled: true}, - SupportCilium: true, - OperatorMetricsEnabled: true, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - scheme := runtime.NewScheme() - require.NoError(t, clientgoscheme.AddToScheme(scheme)) - require.NoError(t, datadoghqv1alpha1.AddToScheme(scheme)) - require.NoError(t, edsdatadoghqv1alpha1.AddToScheme(scheme)) - - mgr, err := ctrl.NewManager(&rest.Config{}, manager.Options{ - Scheme: scheme, - LeaderElection: false, - Controller: ctrlconfig.Controller{ - SkipNameValidation: ptr.To(true), - }, - }) - require.NoError(t, err) - - reconciler := &DatadogAgentInternalReconciler{ - Client: mgr.GetClient(), - PlatformInfo: kubernetes.PlatformInfo{}, - Scheme: scheme, - Recorder: mgr.GetEventRecorderFor("datadogagentinternal-test"), - Options: test.options, - } - require.NoError(t, reconciler.SetupWithManager(mgr, noopMetricsForwardersManager{})) - require.NotNil(t, reconciler.internal) - }) - } -} - -func TestPreparedRolloutPodPredicate(t *testing.T) { - predicate := preparedRolloutPodPredicate() - assert.False(t, predicate.Create(event.CreateEvent{Object: &corev1.ConfigMap{}})) - oldPod := &corev1.Pod{} - assert.True(t, predicate.Create(event.CreateEvent{Object: oldPod})) - readyPod := oldPod.DeepCopy() - readyPod.Status.Conditions = append(readyPod.Status.Conditions, corev1.PodCondition{Type: corev1.PodReady, Status: corev1.ConditionTrue}) - assert.True(t, predicate.Update(event.UpdateEvent{ObjectOld: oldPod, ObjectNew: readyPod}), "PodReady transitions must enqueue capacity fallback") - - waiting := readyPod.DeepCopy() - waiting.Status.ContainerStatuses = []corev1.ContainerStatus{{Name: "agent", Started: ptr.To(false), State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}} - prepared := waiting.DeepCopy() - prepared.Status.ContainerStatuses[0].Started = ptr.To(true) - assert.True(t, predicate.Update(event.UpdateEvent{ObjectOld: waiting, ObjectNew: prepared}), "startup-probe transitions must enqueue capacity fallback") - assert.False(t, predicate.Update(event.UpdateEvent{ObjectOld: &corev1.ConfigMap{}, ObjectNew: &corev1.ConfigMap{}})) - assert.True(t, predicate.Delete(event.DeleteEvent{Object: oldPod})) - assert.False(t, predicate.Generic(event.GenericEvent{Object: oldPod})) -} - -func TestContainerRolloutStatusChanged(t *testing.T) { - running := &corev1.ContainerStateRunning{} - terminated := &corev1.ContainerStateTerminated{} - tests := []struct { - name string - old []corev1.ContainerStatus - new []corev1.ContainerStatus - want bool - }{ - {name: "identical empty", want: false}, - {name: "length", new: []corev1.ContainerStatus{{Name: "agent"}}, want: true}, - {name: "name", old: []corev1.ContainerStatus{{Name: "agent"}}, new: []corev1.ContainerStatus{{Name: "trace-agent"}}, want: true}, - {name: "started", old: []corev1.ContainerStatus{{Name: "agent"}}, new: []corev1.ContainerStatus{{Name: "agent", Started: ptr.To(false)}}, want: true}, - {name: "restart", old: []corev1.ContainerStatus{{Name: "agent"}}, new: []corev1.ContainerStatus{{Name: "agent", RestartCount: 1}}, want: true}, - {name: "running", old: []corev1.ContainerStatus{{Name: "agent"}}, new: []corev1.ContainerStatus{{Name: "agent", State: corev1.ContainerState{Running: running}}}, want: true}, - {name: "terminated", old: []corev1.ContainerStatus{{Name: "agent"}}, new: []corev1.ContainerStatus{{Name: "agent", State: corev1.ContainerState{Terminated: terminated}}}, want: true}, - {name: "identical", old: []corev1.ContainerStatus{{Name: "agent", Started: ptr.To(true), RestartCount: 1, State: corev1.ContainerState{Running: running}}}, new: []corev1.ContainerStatus{{Name: "agent", Started: ptr.To(true), RestartCount: 1, State: corev1.ContainerState{Running: running}}}, want: false}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.want, containerRolloutStatusChanged(test.old, test.new)) - }) - } -} - -func TestPodConditionChanged(t *testing.T) { - empty := &corev1.Pod{} - ready := &corev1.Pod{Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionFalse, Reason: "Starting"}}}} - assert.False(t, podConditionChanged(empty, empty, corev1.PodReady)) - assert.True(t, podConditionChanged(empty, ready, corev1.PodReady)) - assert.False(t, podConditionChanged(ready, ready.DeepCopy(), corev1.PodReady)) - for _, mutate := range []func(*corev1.PodCondition){ - func(condition *corev1.PodCondition) { condition.Status = corev1.ConditionTrue }, - func(condition *corev1.PodCondition) { condition.Reason = "Ready" }, - func(condition *corev1.PodCondition) { condition.Message = "healthy" }, - } { - changed := ready.DeepCopy() - mutate(&changed.Status.Conditions[0]) - assert.True(t, podConditionChanged(ready, changed, corev1.PodReady)) - } -} - -func TestDatadogAgentInternalEventPredicate(t *testing.T) { - p := datadogAgentInternalEventPredicate() - old := &datadoghqv1alpha1.DatadogAgentInternal{ObjectMeta: metav1.ObjectMeta{Generation: 1, Annotations: map[string]string{"example.com/ignored": "old"}}} - - generation := old.DeepCopy() - generation.Generation = 2 - assert.True(t, p.Update(event.UpdateEvent{ObjectOld: old, ObjectNew: generation})) - - prepared := old.DeepCopy() - prepared.Annotations[preparedRolloutModeAnnotationKey] = "prepared-surge-v1" - assert.True(t, p.Update(event.UpdateEvent{ObjectOld: old, ObjectNew: prepared})) - - unrelated := old.DeepCopy() - unrelated.Annotations["example.com/ignored"] = "new" - assert.False(t, p.Update(event.UpdateEvent{ObjectOld: old, ObjectNew: unrelated})) -} - -func TestEnqueueDatadogAgentInternalForPodFollowsDaemonSetOwner(t *testing.T) { - scheme := runtime.NewScheme() - require.NoError(t, appsv1.AddToScheme(scheme)) - ds := &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{ - Name: "profile-agent", - Namespace: "default", - UID: types.UID("ds-uid"), - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: datadoghqv1alpha1.GroupVersion.String(), - Kind: "DatadogAgentInternal", - Name: "profile-ddai", - UID: types.UID("ddai-uid"), - Controller: ptr.To(true), - }}, - }, Spec: appsv1.DaemonSetSpec{Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{preparedRolloutModeAnnotationKey: "prepared-surge-v1"}}}}} - reader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(ds).Build() - pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ - Name: "profile-agent-new", - Namespace: "default", - Labels: map[string]string{apicommon.AgentDeploymentComponentLabelKey: constants.DefaultAgentResourceSuffix}, - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: appsv1.SchemeGroupVersion.String(), - Kind: "DaemonSet", - Name: ds.Name, - UID: ds.UID, - Controller: ptr.To(true), - }}, - }} - - requests := enqueueDatadogAgentInternalForPod(reader)(context.Background(), pod) - require.Len(t, requests, 1) - assert.Equal(t, "default", requests[0].Namespace) - assert.Equal(t, "profile-ddai", requests[0].Name) - - ordinaryDS := ds.DeepCopy() - ordinaryDS.Name = "ordinary-agent" - ordinaryDS.UID = "ordinary-ds-uid" - ordinaryDS.Spec.Template.Annotations = nil - ordinaryReader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(ordinaryDS).Build() - ordinaryPod := pod.DeepCopy() - ordinaryPod.OwnerReferences[0].Name = ordinaryDS.Name - ordinaryPod.OwnerReferences[0].UID = ordinaryDS.UID - assert.Empty(t, enqueueDatadogAgentInternalForPod(ordinaryReader)(context.Background(), ordinaryPod), "ordinary Agent Pods must not trigger prepared-rollout reconciles") - - pod.OwnerReferences[0].UID = "wrong-uid" - assert.Empty(t, enqueueDatadogAgentInternalForPod(reader)(context.Background(), pod), "stale Pod owner UIDs must not enqueue") - - assert.Empty(t, enqueueDatadogAgentInternalForPod(reader)(context.Background(), &corev1.ConfigMap{})) - withoutLabel := pod.DeepCopy() - withoutLabel.Labels = nil - assert.Empty(t, enqueueDatadogAgentInternalForPod(reader)(context.Background(), withoutLabel)) - withoutOwner := pod.DeepCopy() - withoutOwner.OwnerReferences = nil - assert.Empty(t, enqueueDatadogAgentInternalForPod(reader)(context.Background(), withoutOwner)) - wrongOwnerKind := pod.DeepCopy() - wrongOwnerKind.OwnerReferences[0].Kind = "Deployment" - assert.Empty(t, enqueueDatadogAgentInternalForPod(reader)(context.Background(), wrongOwnerKind)) - missingDaemonSet := pod.DeepCopy() - missingDaemonSet.OwnerReferences[0].Name = "missing" - assert.Empty(t, enqueueDatadogAgentInternalForPod(reader)(context.Background(), missingDaemonSet)) - - dsWithoutOwner := ds.DeepCopy() - dsWithoutOwner.Name = "unowned-agent" - dsWithoutOwner.UID = "unowned-ds-uid" - dsWithoutOwner.OwnerReferences = nil - unownedReader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(dsWithoutOwner).Build() - unownedPod := pod.DeepCopy() - unownedPod.OwnerReferences[0].Name = dsWithoutOwner.Name - unownedPod.OwnerReferences[0].UID = dsWithoutOwner.UID - assert.Empty(t, enqueueDatadogAgentInternalForPod(unownedReader)(context.Background(), unownedPod)) -} - func TestEnqueueIfOwnedByDatadogAgentInternal(t *testing.T) { unmanaged := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{ kubernetes.AppKubernetesPartOfLabelKey: "default-profile--ddai", @@ -251,6 +27,6 @@ func TestEnqueueIfOwnedByDatadogAgentInternal(t *testing.T) { managed.Labels[kubernetes.AppKubernetesManageByLabelKey] = "datadog-operator" requests := enqueueIfOwnedByDatadogAgentInternal(context.Background(), managed) require.Len(t, requests, 1) - assert.Equal(t, "default", requests[0].Namespace) - assert.Equal(t, "profile-ddai", requests[0].Name) + owner := object.PartOfLabelValue{Value: "default-profile--ddai"} + assert.Equal(t, owner.NamespacedName(), requests[0].NamespacedName) } diff --git a/internal/controller/testutils/renderer/render_e2e_test.go b/internal/controller/testutils/renderer/render_e2e_test.go index 0dd1193eca..56fa0e1e9b 100644 --- a/internal/controller/testutils/renderer/render_e2e_test.go +++ b/internal/controller/testutils/renderer/render_e2e_test.go @@ -10,6 +10,7 @@ import ( "testing" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" @@ -216,13 +217,18 @@ func TestRender_PreparedRolloutUsesNativeSurge(t *testing.T) { assert.Equal(t, intstr.FromInt(1), *prepared.Spec.UpdateStrategy.RollingUpdate.MaxSurge) assert.Equal(t, intstr.FromInt(0), *prepared.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable) assert.Equal(t, "prepared-surge-v1", prepared.Spec.Template.Annotations["experimental.agent.datadoghq.com/node-agent-rollout-mode"]) + baselineByName := make(map[string]corev1.Container, len(baseline.Spec.Template.Spec.Containers)) + for _, container := range baseline.Spec.Template.Spec.Containers { + baselineByName[container.Name] = container + } preparedPortCount := 0 for _, container := range prepared.Spec.Template.Spec.Containers { preparedPortCount += len(container.Ports) - require.NotNil(t, container.StartupProbe) - require.NotNil(t, container.StartupProbe.Exec) - require.NotNil(t, container.ReadinessProbe) - require.NotNil(t, container.ReadinessProbe.Exec) + baselineContainer, found := baselineByName[container.Name] + require.True(t, found) + assert.Equal(t, baselineContainer.StartupProbe, container.StartupProbe) + assert.Equal(t, baselineContainer.LivenessProbe, container.LivenessProbe) + assert.Equal(t, baselineContainer.ReadinessProbe, container.ReadinessProbe) if container.Name == "trace-agent" { assert.Equal(t, "trace-agent", container.Command[0], "prepared mode must bypass trace-loader") } From 6f8af441d40892814791b1a6f154e33cb156af04 Mon Sep 17 00:00:00 2001 From: "ali.benabdallah" Date: Tue, 28 Jul 2026 14:01:46 +0200 Subject: [PATCH 13/16] Fix resource fallback lint errors --- .../datadogagentinternal/resource_fallback.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/controller/datadogagentinternal/resource_fallback.go b/internal/controller/datadogagentinternal/resource_fallback.go index 8395a09c95..09b9d28feb 100644 --- a/internal/controller/datadogagentinternal/resource_fallback.go +++ b/internal/controller/datadogagentinternal/resource_fallback.go @@ -56,23 +56,23 @@ func (r *Reconciler) reconcilePreparedRollout(ctx context.Context, ddai *datadog if ds.Status.DesiredNumberScheduled <= 0 || ds.Status.ObservedGeneration != ds.Generation { return resourceFallbackPollResult(ds), nil } - budget, err := intstr.GetScaledValueFromIntOrPercent(&budgetValue, int(ds.Status.DesiredNumberScheduled), true) - if err != nil { - return reconcile.Result{}, fmt.Errorf("resolve Agent resource fallback budget: %w", err) + budget, budgetErr := intstr.GetScaledValueFromIntOrPercent(&budgetValue, int(ds.Status.DesiredNumberScheduled), true) + if budgetErr != nil { + return reconcile.Result{}, fmt.Errorf("resolve Agent resource fallback budget: %w", budgetErr) } if budget <= 0 { return reconcile.Result{}, nil } - pods, err := daemonSetPods(ctx, reader, ds) - if err != nil { - return reconcile.Result{}, err + pods, podsErr := daemonSetPods(ctx, reader, ds) + if podsErr != nil { + return reconcile.Result{}, podsErr } if consumedPreparedRolloutBudget(ds, pods) >= budget { return resourceFallbackPollResult(ds), nil } - desiredRevision, err := currentDaemonSetRevision(ctx, reader, ds) - if err != nil { - return reconcile.Result{}, err + desiredRevision, revisionErr := currentDaemonSetRevision(ctx, reader, ds) + if revisionErr != nil { + return reconcile.Result{}, revisionErr } if desiredRevision == "" { return resourceFallbackPollResult(ds), nil From 315585d4ee2d1e47b990af575cecd31c389d9c68 Mon Sep 17 00:00:00 2001 From: "ali.benabdallah" Date: Tue, 28 Jul 2026 14:06:06 +0200 Subject: [PATCH 14/16] Sync Operator workspace modules --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a400500efe..20a049490c 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( k8s.io/apimachinery v0.35.3 k8s.io/cli-runtime v0.35.3 k8s.io/client-go v0.35.3 - k8s.io/component-helpers v0.35.3 + k8s.io/component-helpers v0.35.3 // indirect k8s.io/klog/v2 v2.140.0 k8s.io/kube-aggregator v0.35.3 k8s.io/kube-openapi v0.0.0-20251125145642-4e65d59e963e From 032129115af4da4bf0f81ee4b1e30b8590f91581 Mon Sep 17 00:00:00 2001 From: "ali.benabdallah" Date: Tue, 28 Jul 2026 15:08:23 +0200 Subject: [PATCH 15/16] Sync generated third-party licenses --- LICENSE-3rdparty.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 0c04bd83e3..8274a59a43 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -265,7 +265,7 @@ core,k8s.io/client-go,Apache-2.0 core,k8s.io/client-go/third_party/forked/golang/template,BSD-3-Clause core,k8s.io/cloud-provider/api,Apache-2.0 core,k8s.io/component-base,Apache-2.0 -core,k8s.io/component-helpers,Apache-2.0 +core,k8s.io/component-helpers/resource,Apache-2.0 core,k8s.io/csi-translation-lib,Apache-2.0 core,k8s.io/klog/v2,Apache-2.0 core,k8s.io/kube-aggregator/pkg/apis/apiregistration,Apache-2.0 From d6d5717819240ea43cedcec7fa7634bd59620cb9 Mon Sep 17 00:00:00 2001 From: "ali.benabdallah" Date: Tue, 28 Jul 2026 15:52:54 +0200 Subject: [PATCH 16/16] Avoid caching Agent Pods for rollout --- pkg/config/config.go | 25 +++++++++------------- pkg/config/config_test.go | 44 ++++----------------------------------- 2 files changed, 14 insertions(+), 55 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 882e4b96e7..d0016e18b1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -130,10 +130,10 @@ func CacheOptions(logger logr.Logger, opts WatchOptions) cache.Options { } } - if opts.DatadogAgentEnabled || opts.DatadogAgentProfileEnabled || opts.UntaintControllerEnabled { - // The Agent, profiles, and untaint controllers need to watch Agent Pods. - // The profiles feature needs node name and labels. The Agent and untaint - // controllers need status for rollout and readiness reconciliation. + if opts.DatadogAgentProfileEnabled || opts.UntaintControllerEnabled { + // For the profiles feature and untaint controller we need to list agent pods. + // The profiles feature needs node name and labels; the untaint controller also needs + // Status.Conditions to check readiness. Pods are watched in DatadogAgent namespace(s). // When untaint is configured to wait for CSI, widen to merged agent+CSI // namespaces and drop the pod informer label filter so CSI node-server pods // (app=datadog-csi-driver-node-server) are cached for dual-readiness untaint. @@ -161,24 +161,19 @@ func CacheOptions(logger logr.Logger, opts WatchOptions) cache.Options { newPod := &corev1.Pod{ TypeMeta: pod.TypeMeta, ObjectMeta: v1.ObjectMeta{ - Namespace: pod.Namespace, - Name: pod.Name, - UID: pod.UID, - Labels: pod.Labels, - OwnerReferences: pod.OwnerReferences, + Namespace: pod.Namespace, + Name: pod.Name, + Labels: pod.Labels, }, Spec: corev1.PodSpec{ NodeName: pod.Spec.NodeName, }, } - newPod.Status.Conditions = pod.Status.Conditions - newPod.Status.Phase = pod.Status.Phase - newPod.Status.InitContainerStatuses = pod.Status.InitContainerStatuses - newPod.Status.ContainerStatuses = pod.Status.ContainerStatuses - // The untaint controller also needs Pod.Status.StartTime for its - // readiness-timeout clock. + // The untaint controller needs Pod.Status.Conditions (readiness check) + // and Pod.Status.StartTime (readiness-timeout clock). if opts.UntaintControllerEnabled { + newPod.Status.Conditions = pod.Status.Conditions newPod.Status.StartTime = pod.Status.StartTime } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 2a5bba8fbc..43805bb02d 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -6,11 +6,8 @@ import ( "testing" "golang.org/x/exp/maps" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -123,7 +120,6 @@ func Test_CacheConfig(t *testing.T) { wantObjectConfig: map[client.Object]objectConfig{ agentObj: {configured: true, namespaces: []string{"system"}}, - podObj: {configured: true, namespaces: []string{"system"}}, csiDriverObj: {configured: true, namespaces: []string{"default"}}, csiDaemonSetObj: {configured: true, namespaces: []string{"system", "default"}}, }, @@ -184,7 +180,7 @@ func Test_CacheConfig(t *testing.T) { }, }, { - name: "Only Agent enabled; Monitor enabled without namespace config. Agent Pods are configured; other CRDs and Nodes are not", + name: "Only Agent enabled; Monitor enabled without namespace config. Other CRDs, Pods, Nodes not configured", watchOptions: WatchOptions{ DatadogAgentEnabled: true, @@ -206,13 +202,13 @@ func Test_CacheConfig(t *testing.T) { monitorObj: {configured: true, namespaces: []string{"datadog"}}, sloObj: {configured: false}, profileObj: {configured: false}, - podObj: {configured: true, namespaces: []string{"agentNs1", "agentNs2"}}, + podObj: {configured: false}, nodeObj: {configured: false}, csiDriverObj: {configured: false}, }, }, { - name: "DAP disabled, Introspection enabled; Node uses nil namespace; Agent Pods are configured, Profiles are not", + name: "DAP disabled, Introspection enabled; Node uses nil namespace; Pods, Profiles are not configured", watchOptions: WatchOptions{ DatadogAgentEnabled: true, @@ -235,7 +231,7 @@ func Test_CacheConfig(t *testing.T) { monitorObj: {configured: false}, sloObj: {configured: false}, profileObj: {configured: false}, - podObj: {configured: true, namespaces: []string{"agentNs1", "agentNs2"}}, + podObj: {configured: false}, nodeObj: {configured: true, namespaces: nil}, csiDriverObj: {configured: false}, }, @@ -295,35 +291,3 @@ func verifyResourceNamespace(t *testing.T, resource client.Object, wantConfig ob } } } - -func TestAgentPodCacheTransformPreservesPreparedRolloutStatus(t *testing.T) { - t.Setenv(AgentWatchNamespaceEnvVar, "datadog-agent") - options := CacheOptions(logf.Log.WithName(t.Name()), WatchOptions{DatadogAgentEnabled: true}) - podConfig, found := options.ByObject[podObj] - require.True(t, found) - require.NotNil(t, podConfig.Transform) - - started := true - input := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "agent", Namespace: "datadog-agent"}, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - InitContainerStatuses: []corev1.ContainerStatus{{ - Name: "init-config", - State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 0}}, - }}, - ContainerStatuses: []corev1.ContainerStatus{{ - Name: "agent", - Started: &started, - State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, - }}, - }, - } - transformedObject, err := podConfig.Transform(input) - require.NoError(t, err) - transformed := transformedObject.(*corev1.Pod) - assert.Equal(t, corev1.PodRunning, transformed.Status.Phase) - require.Len(t, transformed.Status.InitContainerStatuses, 1) - require.Len(t, transformed.Status.ContainerStatuses, 1) - assert.True(t, *transformed.Status.ContainerStatuses[0].Started) -}