[johnmave126]

c3b1560 was introduced to optimize waiter callback registration with AF_XDP. The patch requires adding the waiter back after every wait_callback(). But when userspace is holding the net_if lock, wait_callback() defers the priming. When userspace releases the lock, ci_netif_unlock_slow() checks the NIC flag, sees
NIC_FLAG_USERSPACE_PRIME, and calls efxdp_ef_eventq_prime() to prime the NIC. However, efxdp_ef_eventq_prime() is empty but still the queue is considered primed. When next poll happens,
tcp_helper_request_wakeup() checks the primed flag, considers it primed, and skips waiter callback installation. This put poll to sleep without corresponding waiter on the socket. The poll only gets woken up when linux_tcp_helper_periodic_timer fires.

This change removes NIC_FLAG_USERSPACE_PRIME for AF_XDP, so polls can install waiter callback correctly.

Minimal reproducer, the sender sends the packet with sequence numbers at a definite rate, and the receiver measures the gap in the sequence number:

udp_sender.c

#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <time.h>
#include <arpa/inet.h>

int main(int argc, char **argv) {
  if (argc < 5) {
    fprintf(stderr, "Usage: %s <ip> <port> <ifname> <pps>\n", argv[0]);
    return 1;
  }
  const char *ip = argv[1];
  int port = atoi(argv[2]);
  const char *ifname = argv[3];
  int pps = atoi(argv[4]);

  int fd = socket(AF_INET, SOCK_DGRAM, 0);
  setsockopt(fd, SOL_SOCKET, SO_BINDTODEVICE, ifname, strlen(ifname));

  struct sockaddr_in addr = {
      .sin_family = AF_INET,
      .sin_port = htons(port),
      .sin_addr.s_addr = inet_addr(ip),
  };

  uint64_t interval_ns = 1000000000ULL / pps;
  struct timespec next;
  clock_gettime(CLOCK_MONOTONIC, &next);

  char buf[64] = {0};
  uint64_t seq = 0;
  while (1) {
    memcpy(buf, &seq, sizeof(seq));
    sendto(fd, buf, sizeof(buf), 0, (struct sockaddr *)&addr, sizeof(addr));
    seq++;

    next.tv_nsec += interval_ns;
    while (next.tv_nsec >= 1000000000L) {
      next.tv_nsec -= 1000000000L;
      next.tv_sec += 1;
    }
    clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &next, NULL);
  }
  return 0;
}

udp_receiver.c

#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <poll.h>
#include <arpa/inet.h>

int main(int argc, char **argv) {
  if (argc < 5) {
    fprintf(stderr, "Usage: %s <ip> <port> <ifname> <rcvbuf_size>\n", argv[0]);
    return 1;
  }
  const char *ip = argv[1];
  int port = atoi(argv[2]);
  const char *ifname = argv[3];
  int rcvbuf = atoi(argv[4]);

  int fd = socket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0);
  int opt = 1;
  setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));
  setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof(rcvbuf));
  setsockopt(fd, SOL_SOCKET, SO_BINDTODEVICE, ifname, strlen(ifname));

  struct sockaddr_in addr = {
      .sin_family = AF_INET,
      .sin_port = htons(port),
      .sin_addr.s_addr = inet_addr(ip),
  };
  bind(fd, (struct sockaddr *)&addr, sizeof(addr));

  uint64_t expected_seq = 0, rx = 0, gaps = 0, dropped = 0;
  struct pollfd pfd = {.fd = fd, .events = POLLIN};
  while (poll(&pfd, 1, rx == 0 ? 30000 : 3000) > 0) {
    uint64_t seq;
    while (recv(fd, &seq, sizeof(seq), 0) > 0) {
      if (rx++ == 0) expected_seq = seq;
      if (seq != expected_seq) {
        gaps++;
        dropped += (seq - expected_seq);
      }
      expected_seq = seq + 1;
      if (rx % 50000 == 0)
        printf("rx=%lu gaps=%lu dropped=%lu\n", rx, gaps, dropped);
    }
  }
  printf("FINAL: rx=%lu gaps=%lu dropped=%lu\n", rx, gaps, dropped);
  return 0;
}

Run with two machines with AF_XDP

• Receiver: onload ./udp_receiver <IP> <PORT> <NIC_NAME> <SO_RCVBUF>
• Sender: onload ./udp_sender <IP> <PORT> <NIC_NAME> <PPS>
• Stop sender when done.

In our testing environment, with 75k pps

• Before patch:

  rx=50000 gaps=97 dropped=514843
  FINAL: rx=67227 gaps=130 dropped=682687
  

  showing 91% packet drop rate.
• After patch:

  rx=50000 gaps=0 dropped=0
  ...
  rx=700000 gaps=0 dropped=0
  FINAL: rx=749886 gaps=0 dropped=0
  

  showing no drop anymore.