test: cover the async delete create event received while CNS runs

hack/scripts/async-delete-test.sh

@@ -48,3 +48,92 @@ do
     fi
 done
 
+# The loop above stops CNS before it deletes the Pod, so the pending delete file
+# is always on disk before CNS starts and CNS reads it in the startup directory
+# scan. When CNS runs but does not answer, the file instead appears while the
+# fsnotify watcher listens. That is a different code path and it must release the
+# IP without a CNS restart. SIGSTOP gives that state: the socket stays open, the
+# CNI request times out after 15s, and the kernel queues the create event until
+# SIGCONT. shareProcessNamespace on the daemonset lets the debug container signal
+# the CNS process.
+echo "verify async delete for a create event received while CNS runs"
+kubectl rollout status deployment busybox
+
+busybox_pod=$(kubectl get pods -l k8s-app=busybox --no-headers -o custom-columns=NAME:.metadata.name | head -1)
+if [ -z "$busybox_pod" ]; then
+    echo "##[error]no busybox pod found"
+    exit 1
+fi
+node_name=$(kubectl get pod $busybox_pod -o jsonpath='{.spec.nodeName}')
+cns_pod=$(kubectl get pods -l k8s-app=azure-cns -n kube-system --field-selector "spec.nodeName=$node_name,status.phase=Running" -o jsonpath='{.items[0].metadata.name}')
+if [ -z "$cns_pod" ]; then
+    echo "##[error]no running CNS pod on node $node_name"
+    exit 1
+fi
+
+cns_pid=$(kubectl exec -i $cns_pod -c debug -n kube-system -- sh -c 'for p in /proc/[0-9]*; do if [ "$(cat $p/comm 2>/dev/null)" = "azure-cns" ]; then echo ${p#/proc/}; fi; done' | tr -d '\r' | head -1)
+if [ -z "$cns_pid" ]; then
+    echo "##[error]azure-cns process not visible from the debug container. is shareProcessNamespace set on the daemonset?"
+    exit 1
+fi
+
+restarts_before=$(kubectl get pod $cns_pod -n kube-system -o jsonpath='{.status.containerStatuses[?(@.name=="cns-container")].restartCount}')
+
+# a paused CNS breaks every later step, so always resume it, on any exit path
+resume_cns() {
+    kubectl exec -i $cns_pod -c debug -n kube-system -- sh -c "kill -CONT $cns_pid" > /dev/null 2>&1
+}
+trap resume_cns EXIT INT TERM
+
+echo "stop the CNS process $cns_pid so the CNI cannot reach it"
+kubectl exec -i $cns_pod -c debug -n kube-system -- sh -c "kill -STOP $cns_pid"
+
+echo "delete busybox pod $busybox_pod. the CNI DEL waits for the 15s CNS request timeout"
+kubectl delete pod $busybox_pod --timeout=120s
+
+echo "check the CNI wrote a pending delete while CNS was stopped"
+if ! pending_file=$(kubectl exec -i $cns_pod -c debug -n kube-system -- ls /var/run/azure-vnet/deleteIDs 2>&1); then
+    echo "##[error]could not list the deleteIDs directory: $pending_file"
+    exit 1
+fi
+
+echo "resume the CNS process"
+kubectl exec -i $cns_pod -c debug -n kube-system -- sh -c "kill -CONT $cns_pid"
+
+if [ -z "$pending_file" ]; then
+    echo "##[error]async delete failure. the CNI wrote no file, so the async delete path did not run."
+    exit 1
+fi
+echo "pending deletes"
+echo "$pending_file"
+
+echo "wait up to 60s for CNS to process the create event"
+waited=0
+drained=false
+while [ $waited -lt 60 ]; do
+    sleep 5s
+    waited=$((waited + 5))
+    # only an empty listing that actually succeeded proves the release happened
+    if live_file=$(kubectl exec -i $cns_pod -c debug -n kube-system -- ls /var/run/azure-vnet/deleteIDs 2>&1); then
+        if [ -z "$live_file" ]; then
+            drained=true
+            break
+        fi
+    else
+        echo "could not list the deleteIDs directory, retrying: $live_file"
+    fi
+done
+
+if [ "$drained" != "true" ]; then
+    echo "##[error]async delete failure. CNS did not process the create event. file still exists in deleteIDs directory."
+    exit 1
+fi
+
+restarts_after=$(kubectl get pod $cns_pod -n kube-system -o jsonpath='{.status.containerStatuses[?(@.name=="cns-container")].restartCount}')
+if [ "$restarts_before" != "$restarts_after" ]; then
+    echo "##[error]async delete failure. CNS restarted, so the startup scan processed the file instead of the create event."
+    exit 1
+fi
+
+echo "async delete success for the create event"
+

test/integration/manifests/cns/daemonset-linux.yaml

@@ -124,6 +124,10 @@ spec:
             - name: cni-conflist
               mountPath: /etc/cni/net.d
       hostNetwork: true
+      # lets the debug container signal the CNS process. async-delete-test.sh
+      # pauses it with SIGSTOP so the CNI request times out while the container
+      # keeps running and the fsnotify watcher stays alive
+      shareProcessNamespace: true
       volumes:
         - name: cni-conflist
           hostPath: