Add AF_XDP sockets

Describe XDP option in help
Add option to link with local libbpf folder
Update wrapper server script
Fix min max functions conflicts
Compile xdp-kern.o
Add IFLAGS to CFLAGS in makefile for XDP
Add bpf_helpers and xdp_kern files
Close XDP socket before exiting program
Add tracecmd option for XDP
Print stats for XDP too
Add an active polling mode

packet-exchange/build/Makefile

@@ -27,15 +27,35 @@ CFLAGS += -MD -MP
 CFLAGS += -I $(SRCDIR)
 CFLAGS += -std=gnu99
 
-LDFLAGS = -pthread
+LLIBS = -pthread
+
+ifneq ($(WITH_XDP),)
+CFLAGS += -D WITH_XDP
+LDFLAGS += -L/usr/lib -lbpf
+else ifneq ($(WITH_GIT_XDP),)
+IFLAGS += -I ${HOME}/libbpf/include
+CFLAGS += -D WITH_XDP $(IFLAGS)
+LDIRS += -L${HOME}/libbpf/src
+LLIBS += -lelf -lz -l:libbpf.a
+endif
 
 vpath %.c $(SRCDIR)
 
+xdp_kern.o: xdp_kern.c
+	clang $(IFLAGS) -isystem /usr/include/arm-linux-gnueabihf -S -target bpf -D __BPF_TRACING__ -Wall -O2 -emit-llvm -c -g -o xdp_kern.ll $^
+	llc -march=bpf -filetype=obj -o $@ xdp_kern.ll
+
+ifneq ($(WITH_GIT_XDP),)
+$(SERVER_PROG): $(SERVER_OBJS) xdp_kern.o
+	$(CC) $(LDFLAGS) $(LDIRS) $(SERVER_OBJS) $(LLIBS) -o $@
+else
 $(SERVER_PROG): $(SERVER_OBJS)
-	$(CC) $(LDFLAGS) $^ -o $@
+	$(CC) $(LDFLAGS) $(LDIRS) $^ $(LLIBS) -o $@
+endif
 
 $(CLIENT_PROG): $(CLIENT_OBJS)
-	$(CC) $(LDFLAGS) $^ -o $@
+	$(CC) $(LDFLAGS) $(LDIRS) $^ $(LLIBS) -o $@
+
 
 -include $(subst .c,.d,$(SERVER_SRCS))
 -include $(subst .c,.d,$(CLIENT_SRCS))

packet-exchange/src/client.c

@@ -90,24 +90,30 @@ static char send_data[MAX_BUFFER_SIZE];
 
 static void help(char *argv[]) {
 	printf(
-      "Usage: %s -f IF  [-abthgv] [-e ETF_OFFSET] [-d BUF_LEN] [-i USEC] [-l N]"
+	    "Usage: %s -f IF  [-abthgv] [-e ETF_OFFSET] [-d BUF_LEN] [-i USEC] "
+	    "[-l N]"
 	    "[-p PRIO] [-q PACKET_PRIO] [-r USEC] [-T LATENCY_THRESHOLD -G]\n\n"
 	    "       -a                    Run the real time thread on CPU1\n"
 	    "       -b                    Measure RTT\n"
 	    "       -d BUF_LEN            Set the length of tx buffer\n"
-      "       -e ETF_OFFSET         Set a txtime with an offset of ETF_OFFSET "
+	    "       -e ETF_OFFSET         Set a txtime with an offset of "
+	    "ETF_OFFSET "
 	    "us (to be used in an ETF qdisc)\n"
-      "       -f IF                 Set the network interface to be used\n"
+	    "       -f IF                 Set the network interface to be "
+	    "used\n"
 	    "       -g                    Print histograms to sdtout on exit\n"
 	    "       -h                    Show help\n"
-      "       -i USEC               Wake up the real time thread every USEC "
+	    "       -i USEC               Wake up the real time thread every "
+	    "USEC "
 	    "microseconds (Default: 10ms)\n"
 	    "       -l N                  Wake up the real time thread N times "
 	    "(Default: 0)\n"
 	    "       -p PRIO               Run the real time thread at priority "
 	    "PRIO\n"
-      "       -q PACKET_PRIO        Send packets with PACKET_PRIO priority\n"
-      "       -r USEC               Refresh the non real time main thread "
+	    "       -q PACKET_PRIO        Send packets with PACKET_PRIO "
+	    "priority\n"
+	    "       -r USEC               Refresh the non real time main "
+	    "thread "
 	    "every USEC microseconds (Default: 50ms)\n"
 	    "       -t                    Enable timestamps\n"
 	    "       -v                    Verbose\n"
@@ -131,7 +137,8 @@ static void *packet_sending_thread(void *p) {
 		CPU_ZERO(&mask);
 		CPU_SET(1, &mask);
 		if (sched_setaffinity(0, sizeof(mask), &mask))
-      error(EXIT_FAILURE, errno, "Could not set CPU affinity to CPU #1\n");
+			error(EXIT_FAILURE, errno,
+			      "Could not set CPU affinity to CPU #1\n");
 	}
 
 	if (enable_etf) {
@@ -164,10 +171,13 @@ static void *packet_sending_thread(void *p) {
 		if (nb_cycles) {
 			int interval_us = calcdiff_ns(current, previous) / 1000;
 
-      egress_stats.min_interval = min(interval_us, egress_stats.min_interval);
-      egress_stats.max_interval = max(interval_us, egress_stats.max_interval);
+			egress_stats.min_interval =
+			    _min_(interval_us, egress_stats.min_interval);
+			egress_stats.max_interval =
+			    _max_(interval_us, egress_stats.max_interval);
 			egress_stats.avg_interval =
-          (egress_stats.avg_interval * nb_cycles + interval_us) /
+			    (egress_stats.avg_interval * nb_cycles +
+			     interval_us) /
 			    (nb_cycles + 1);
 		}
 
@@ -207,7 +217,8 @@ int main(int argc, char *argv[]) {
 	egress_params.packet_priority = 3;
 	egress_params.tx_buffer_len = 1024;
 
-  /* Lock all current and future pages from preventing of being paged to swap */
+	/* Lock all current and future pages from preventing of being paged to
+	 * swap */
 	if (mlockall(MCL_CURRENT | MCL_FUTURE)) {
 		perror("mlockall failed");
 		/* exit(-1) or do error handling */
@@ -231,16 +242,16 @@ int main(int argc, char *argv[]) {
 	}
 
 	// Catch breaks with sighand to print the histograms
-  init_signals(sighand, enable_histograms);
+	init_signals(sighand);
 
 	// Initialize the UDP packet sending socket
-  init_udp_send(&egress_params, &egress_stats, enable_histograms, main_params.enable_tracing,
-                kernel_latency_hist);
+	init_udp_send(&egress_params, &egress_stats, enable_histograms,
+		      main_params.enable_tracing, kernel_latency_hist);
 
 	// Initialize the UDP packet receiving socket if RTT is measured
 	if (tsn_task == RTT_TASK)
-    init_udp_recv(&ingress_params, &ingress_stats, enable_histograms,
-                  kernel_latency_hist);
+		init_udp_recv(&ingress_params, &ingress_stats,
+			      enable_histograms, kernel_latency_hist);
 
 	/* Initialize pthread attributes (default values) */
 	if (pthread_attr_init(&attr)) {
@@ -270,7 +281,8 @@ int main(int argc, char *argv[]) {
 
 	// Create the real time thread
 	if (pthread_create(&thread, &attr, packet_sending_thread, NULL))
-    error(EXIT_FAILURE, errno, "Couldn't create packet sending thread");
+		error(EXIT_FAILURE, errno,
+		      "Couldn't create packet sending thread");
 
 	// Verbose loop
 	for (;;) {
@@ -278,21 +290,27 @@ int main(int argc, char *argv[]) {
 
 		if (main_params.verbose) {
 			if (tsn_task == RTT_TASK) {
-        printf("%9" PRIu64 ": RTT: %4d %4d %4d\n", nb_cycles, rtt_stats.min_rtt,
+				printf("%9" PRIu64 ": RTT: %4d %4d %4d\n",
+				       nb_cycles, rtt_stats.min_rtt,
 				       rtt_stats.avg_rtt, rtt_stats.max_rtt);
 				printf("\033[%dA", 1);
 			} else {
-        printf("%9" PRIu64 ": [%4d, %4d], I (10us): %3d %3d %3d", nb_cycles,
+				printf("%9" PRIu64
+				       ": [%4d, %4d], I (10us): %3d %3d %3d",
+				       nb_cycles,
 				       (int)egress_stats.invalid_parameter,
 				       (int)egress_stats.missed_deadline,
-               egress_stats.min_interval / 10, egress_stats.avg_interval / 10,
+				       egress_stats.min_interval / 10,
+				       egress_stats.avg_interval / 10,
 				       egress_stats.max_interval / 10);
 
 				if (enable_timestamps) {
-          printf(", K: %4d %4d %4d [%4d]\n", egress_stats.min_kernel_latency,
+					printf(", K: %4d %4d %4d [%4d]\n",
+					       egress_stats.min_kernel_latency,
 					       egress_stats.avg_kernel_latency,
 					       egress_stats.max_kernel_latency,
-                 (int)egress_stats.high_kernel_latency);
+					       (int)egress_stats
+						   .high_kernel_latency);
 				} else {
 					printf("\n");
 				}
@@ -322,19 +340,21 @@ static void do_tsn_task(char *data, uint64_t next_txtime) {
 	} else if (tsn_task == RTT_TASK) {
 		clock_gettime(CLOCK_MONOTONIC, &t1);
 		send_udp_packet(data, next_txtime);
-    recv_udp_packet(0, 0, NULL);
+		recv_udp_packet();
 		clock_gettime(CLOCK_MONOTONIC, &t2);
 
 		rtt_us = calcdiff_ns(t2, t1) / 1000;
 
-    rtt_stats.min_rtt = min(rtt_us, rtt_stats.min_rtt);
-    rtt_stats.max_rtt = max(rtt_us, rtt_stats.max_rtt);
+		rtt_stats.min_rtt = _min_(rtt_us, rtt_stats.min_rtt);
+		rtt_stats.max_rtt = _max_(rtt_us, rtt_stats.max_rtt);
 		rtt_stats.avg_rtt =
-        (((uint64_t)rtt_stats.avg_rtt) * (nb_cycles - 1) + rtt_us) / nb_cycles;
+		    (((uint64_t)rtt_stats.avg_rtt) * (nb_cycles - 1) + rtt_us) /
+		    nb_cycles;
 
 		if (rtt_us > MAX_RTT)
-      fprintf(stderr, "RTT value higher than MAX_RTT : %d ( > %d)\n", rtt_us,
-              MAX_RTT);
+			fprintf(stderr,
+				"RTT value higher than MAX_RTT : %d ( > %d)\n",
+				rtt_us, MAX_RTT);
 		else
 			rtt_hist[rtt_us]++;
 	}
@@ -364,7 +384,8 @@ static void print_histograms() {
 	    "\"metadata\": {\"i\": \"%dus\", \"duration\": \"%dh%d\", "
 	    "\"etf_offset\": \"%dus\"},"
 	    "\"props\": [[",
-      (tsn_task == SEND_PACKET_TASK) ? "packet_send_timestamps" : "packet_rtt",
+	    (tsn_task == SEND_PACKET_TASK) ? "packet_send_timestamps"
+					   : "packet_rtt",
 	    (tsn_task == SEND_PACKET_TASK) ? "kernel_space" : "rtt", interval,
 	    duration_hour, duration_minutes, thread_params.etf_offset);
 
@@ -377,13 +398,14 @@ static void print_histograms() {
 	}
 
 	for (int j = 0; j < max_hist_val; j++)
-    printf("%" PRIi64 "%s", histogram[j], (j + 1 < max_hist_val ? ", " : ""));
+		printf("%" PRIi64 "%s", histogram[j],
+		       (j + 1 < max_hist_val ? ", " : ""));
 	printf("]]}]}\n");
 }
 
 static void sighand(int sig_num) {
 	(void)sig_num;
-  print_histograms();
+	if (enable_histograms) print_histograms();
 	exit(EXIT_SUCCESS);
 }
 
@@ -406,7 +428,9 @@ static void process_options(int argc, char *argv[]) {
 			case 'd':
 				egress_params.tx_buffer_len = atoi(optarg);
 				if (egress_params.tx_buffer_len < 1) {
-          fprintf(stderr, "BUF_LEN should be greater than 1\n");
+					fprintf(stderr,
+						"BUF_LEN should be greater "
+						"than 1\n");
 					exit(EXIT_FAILURE);
 				}
 				break;

packet-exchange/src/common.c

 #define _GNU_SOURCE
+#include "common.h"
+
 #include <inttypes.h>
 #include <signal.h>
 #include <stdint.h>
@@ -7,8 +9,6 @@
 #include <time.h>
 #include <unistd.h>
 
-#include "common.h"
-
 void (*previous_handlers[NSIG])(int);
 static void (*sighand)(int);
 
@@ -32,30 +32,27 @@ uint64_t calcdiff_ns(struct timespec t1, struct timespec t2) {
 	return diff;
 }
 
-int max(int a, int b) { return a > b ? a : b; }
-int min(int a, int b) { return a < b ? a : b; }
+int _max_(int a, int b) { return a > b ? a : b; }
+int _min_(int a, int b) { return a < b ? a : b; }
 
-int histogram_min(uint64_t * histogram, int max_value) {
+int histogram_min(uint64_t *histogram, int max_value) {
 	int ret = max_value;
-  for (int i = max_value; i >= 0; i--)
-    ret = histogram[i] ? i : ret;
+	for (int i = max_value; i >= 0; i--) ret = histogram[i] ? i : ret;
 	return ret;
 }
-int histogram_max(uint64_t * histogram, int max_value) {
+int histogram_max(uint64_t *histogram, int max_value) {
 	int ret = 0;
-  for (int i = 0; i <= max_value; i++)
-    ret = histogram[i] ? i : ret;
+	for (int i = 0; i <= max_value; i++) ret = histogram[i] ? i : ret;
 	return ret;
 }
 
-
 static void sighand_wrapper(int sig) {
-
 	// If we get un unexpected signal, report it, if not print the histogram
 	if (sig == SIGINT || sig == SIGTERM)
 		(*sighand)(sig);  // Will print the histogram
 	else
-    printf("Uknown signal interrupt: %s (%d)\n", strsignal(sig), sig);
+		printf("Uknown signal interrupt: %s (%d)\n", strsignal(sig),
+		       sig);
 
 	// Execute the default handler
 	if (previous_handlers[sig] == SIG_DFL) {
@@ -68,12 +65,9 @@ static void sighand_wrapper(int sig) {
 	}
 }
 
-void init_signals(void (*_sighand)(int), int enable_histograms) {
-
+void init_signals(void (*_sighand)(int)) {
 	sighand = _sighand;
 
-  if (enable_histograms)
-    for (int i = 0; i < NSIG; i++)
-      signal(i, sighand_wrapper);
+	for (int i = 0; i < NSIG; i++) signal(i, sighand_wrapper);
 }
 

packet-exchange/src/common.h

@@ -8,6 +8,17 @@
 #include <time.h>
 #include <unistd.h>
 
+#ifdef WITH_XDP
+#include <bpf/bpf.h>
+#include <bpf/libbpf.h>
+#include <bpf/xsk.h>
+#include <linux/if_ether.h>
+#include <linux/if_link.h>
+#include <linux/if_xdp.h>
+#include <linux/ip.h>
+#include <linux/udp.h>
+#endif
+
 #define NSEC_PER_SEC UINT64_C(1000000000)
 #define SERVER_PORT "50000"
 #define SERVER_PORT_INT 50000
@@ -19,18 +30,25 @@
 #define MAX_BUFFER_SIZE 1024
 
 #define TIMESTAMP_BUFFER_SIZE 4096
+#define err(...)                              \
+	do {                                  \
+		fprintf(stderr, __VA_ARGS__); \
+		exit(EXIT_FAILURE);           \
+	} while (0)
+
+#define err_errno(...) error(EXIT_FAILURE, errno, __VA_ARGS__);
 
 uint64_t ts_to_uint(struct timespec t);
 void add_ns(struct timespec *t, uint64_t ns);
 uint64_t calcdiff_ns(struct timespec t1, struct timespec t2);
 
-void init_signals(void (*_sighand)(int), int enable_histograms);
+void init_signals(void (*_sighand)(int));
 
-int min(int a, int b);
-int max(int a, int b);
+int _min_(int a, int b);
+int _max_(int a, int b);
 
-int histogram_min(uint64_t * histogram, int max_value);
-int histogram_max(uint64_t * histogram, int max_value);
+int histogram_min(uint64_t *histogram, int max_value);
+int histogram_max(uint64_t *histogram, int max_value);
 
 extern void (*previous_handlers[NSIG])(int);
 

packet-exchange/src/recv_packet.h

@@ -7,13 +7,14 @@ typedef struct ingress_param {
 	char network_if[16];
 
 	int use_timestamps;
+	int xdp_polling_mode;
+	int interval;
 
 	size_t tx_buffer_len;
 
 } ingress_param_t;
 
 typedef struct ingress_stat {
-
 	int min_kernel_latency;
 	int avg_kernel_latency;
 	int max_kernel_latency;
@@ -30,11 +31,34 @@ typedef struct ingress_stat {
 	char data[MAX_BUFFER_SIZE];
 } ingress_stat_t;
 
-void init_udp_recv(ingress_param_t *_params,
-                   ingress_stat_t *stats,
-                   int use_histogram,
-                   uint64_t *_kernel_latency_hist);
+void init_udp_recv(ingress_param_t *_params, ingress_stat_t *stats,
+		   int use_histogram, uint64_t *_kernel_latency_hist);
+
+void recv_udp_packet(void);
+
+#ifdef WITH_XDP
+#define NUM_FRAMES 4096
+#define FRAME_SIZE XSK_UMEM__DEFAULT_FRAME_SIZE
+
+struct xsk_umem_info {
+	struct xsk_ring_prod fq;
+	struct xsk_ring_cons cq;
+	struct xsk_umem *umem;
+	void *buffer;
+};
+struct xdpsock {
+	struct xsk_ring_cons rx;
+	struct xsk_ring_prod tx;
+	struct xsk_umem_info umem;
+	struct xsk_socket *xsk;
+	int fd;
+};
+#endif
 
-void recv_udp_packet();
+void init_xdp_recv(ingress_param_t * _params);
+int recv_xdp_packet(void);
+void recv_xdp_cleanup(void);
+void setup_poll_fd(void);
+void close_xdp_socket(void);
 
 #endif

packet-exchange/src/send_packet.c

@@ -222,9 +222,9 @@ static void process_error_queue() {
       ts_buf_read_index = (ts_buf_read_index + 1) % TIMESTAMP_BUFFER_SIZE;
 
       stats->min_kernel_latency =
-          min(kernel_latency, stats->min_kernel_latency);
+          _min_(kernel_latency, stats->min_kernel_latency);
       stats->max_kernel_latency =
-          max(kernel_latency, stats->max_kernel_latency);
+          _max_(kernel_latency, stats->max_kernel_latency);
       stats->avg_kernel_latency =
           (stats->max_kernel_latency * packets_sent + kernel_latency) /
           (packets_sent + 1);

packet-exchange/src/tracer.c

@@ -75,7 +75,7 @@ static void setkernvar(const char *name, char *value) {
     fprintf(stderr, "could not set %s to %s\n", name, value);
 }
 
-void open_fds() {
+void open_fds(void) {
 
   fileprefix = debugfileprefix;
 

packet-exchange/src/tracer.h

@@ -2,7 +2,7 @@
 #define TRACER_H
 
 void setup_tracer(int enable_graph);
-void open_fds();
+void open_fds(void);
 void tracing(int on);
 void tracemark(char * s);
 

packet-exchange/src/xdp_kern.c

+#define KBUILD_MODNAME "blub"
+
+#include <linux/bpf.h>
+#include <linux/if_ether.h>
+#include <linux/if_link.h>
+#include <linux/if_xdp.h>
+#include <linux/in.h>
+#include <linux/ip.h>
+#include <linux/udp.h>
+
+#include "bpf_helpers.h"
+
+#define bpf_printk(fmt, ...)                                               \
+	({                                                                 \
+		char ____fmt[] = fmt;                                      \
+		bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \
+	})
+
+struct bpf_map_def SEC("maps") xsks_map = {
+    .type = BPF_MAP_TYPE_XSKMAP,
+    .key_size = sizeof(int),
+    .value_size = sizeof(int),
+    .max_entries = 64,
+};
+
+static inline int parse_ipv4(void *data, unsigned long long nh_off,
+			     void *data_end) {
+	struct iphdr *iph = data + nh_off;
+
+	if ((void *)(iph + 1) > data_end) return 0;
+
+	return iph->protocol;
+}
+
+SEC("xdp_sock")
+int xdp_sock_prog(struct xdp_md *ctx) {
+	void *data_end = (void *)(long)ctx->data_end;
+	void *data = (void *)(long)ctx->data;
+	struct ethhdr *eth = data;
+	int idx = ctx->rx_queue_index;
+	unsigned int ipproto = 0;
+	unsigned long long nh_off;
+
+	/* Check if it's a UDP frame: If UDP -> Redirect to active xsk for user
+	 * space. If not -> pass to stack.
+	 */
+	nh_off = sizeof(*eth);
+	if (data + nh_off > data_end) return XDP_PASS;
+
+	if (eth->h_proto == __builtin_bswap16(ETH_P_IP))
+		ipproto = parse_ipv4(data, nh_off, data_end);
+
+	if (ipproto != IPPROTO_UDP) return XDP_PASS;
+
+	/* If socket bound to rx_queue then redirect to user space */
+	if (bpf_map_lookup_elem(&xsks_map, &idx))
+		return bpf_redirect_map(&xsks_map, idx, 0);
+
+	/* Else pass to Linux' network stack */
+	return XDP_PASS;
+}
+
+char _license[] SEC("license") = "GPL";
+

scripts/run-server

@@ -3,16 +3,28 @@
 script_dir=$(dirname $(realpath $0))
 
 usage() {
-  echo "Usage: $0 [[-bt] [-g INTERVAL]] | (-d NB_PACKETS [-i INTERVAL]) [-T LATENCY_THRESHOLD -G]" 1>&2;
+  echo "Usage: $0  |  " 1>&2;
+  exit 1;
+}
+usage() {
+  cat << ENDUSAGE
+Usage: $0 SERVER | TCPDUMP [TRACE_OPTS]
+   SERVER: [-bt] [(-x | -X) POLL] [-g INTERVAL]
+   TCPDUMP: -d NB_PACKETS [-i INTERVAL]
+   TRACE_OPTS: (-T LATENCY_THRESHOLD -G) | -E
+ENDUSAGE
+1>&2;
+
   exit 1;
 }
 
 # Default options
 server_options="-a -p 99 -f eth0"
+make_opts=""
 ip="10.100.21."
 tcpdump_interval=1000000
 
-while getopts "b:htd:i:g:T:G" opt; do
+while getopts "b:htx:X:d:i:g:T:GE" opt; do
   case "${opt}" in
     b )
         use_rtt=1
@@ -37,12 +49,23 @@ while getopts "b:htd:i:g:T:G" opt; do
     t )
         server_options+=" -t"
         ;;
+    x )
+        server_options+=" -x ${OPTARG}"
+	make_opts=" -e WITH_XDP=1"
+        ;;
+    X )
+        server_options+=" -x ${OPTARG}"
+	make_opts=" -e WITH_GIT_XDP=1"
+        ;;
     T )
         server_options+=" -T ${OPTARG}"
       ;;
     G )
         server_options+=" -G"
       ;;
+    E )
+        enable_xdp_events=1
+      ;;
     * )
         usage
       ;;
@@ -78,10 +101,14 @@ if [ -n "${use_tcpdump}" ]; then
   $script_dir/txtime-stats.py -f server_stats_tmp.out -i $tcpdump_interval;
 else
 
-  echo "make server";
-  cd $script_dir/../packet-exchange/build;make server;cd $script_dir
+  echo "make $make_opts server";
+  cd $script_dir/../packet-exchange/build;make $make_opts server;cd $script_dir
 
-  if [ -z "${use_histogram}" ]; then
+  if [ -n "${enable_xdp_events}" ]; then
+    tracecmd_events="-e irq -e sched -e xdp"
+    echo "trace-cmd record $tracecmd_events $script_dir/../packet-exchange/build/server $server_options";
+    trace-cmd record $tracecmd_events $script_dir/../packet-exchange/build/server $server_options;
+  elif [ -z "${use_histogram}" ]; then
     echo "server $server_options";
     $script_dir/../packet-exchange/build/server $server_options;
   else