Properly compute userspace and kernelspace timestamps when receiving a packet

packet-exchange/src/client.c

@@ -207,13 +207,13 @@ int main(int argc, char *argv[]) {
 
         printf("(%d)  Enter send_udp_packet timestamp: %" PRIu64 "\n",
                stats->nb_cycles,
-               stats->packet_ts.enter_user_space);
+               stats->packet_ts.userspace_enter_ts);
         printf("(%d)  Call sendmsg timestamp         : %" PRIu64 "\n",
                stats->nb_cycles,
-               stats->packet_ts.enter_kernel);
+               stats->packet_ts.userspace_exit_ts);
         printf("(%d)  Leave kernel timestamp         : %" PRIu64 "\n",
                stats->nb_cycles,
-               stats->packet_ts.leave_kernel);
+               stats->packet_ts.kernelspace_ts);
       }
     }
 

packet-exchange/src/recv_packet.c

@@ -122,12 +122,6 @@ packet_timestamps_t recv_udp_packet(int use_timestamps, int use_histograms, int6
   packet_timestamps_t packet_ts;
   struct timespec ts;
 
-
-  if (use_timestamps) {
-    clock_gettime(CLOCK_REALTIME, &ts);
-    packet_ts.enter_user_space = ts_to_uint(ts);
-  }
-
   iov.iov_base = &rx_buffer;
   iov.iov_len = BUFFER_SIZE - 1;
 
@@ -141,31 +135,42 @@ packet_timestamps_t recv_udp_packet(int use_timestamps, int use_histograms, int6
 
   if (use_timestamps) {
     clock_gettime(CLOCK_REALTIME, &ts);
-    packet_ts.enter_kernel = ts_to_uint(ts);
+    packet_ts.userspace_exit_ts = ts_to_uint(ts);
   }
 
   recvmsgerr = recvmsg(sock_fd, &msg, 0);
   if (recvmsgerr < 0)
     error(EXIT_FAILURE, errno, "recvmsg failed, ret value: %d\n", recvmsgerr);
 
+  if (use_timestamps) {
+    clock_gettime(CLOCK_REALTIME, &ts);
+    packet_ts.userspace_enter_ts = ts_to_uint(ts);
+  }
+
   if(use_timestamps) {
     for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
       if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SO_TIMESTAMPING) {
 
         struct timespec *stamp = (struct timespec *)CMSG_DATA(cmsg);
-        packet_ts.leave_kernel = ts_to_uint(*stamp);
+        packet_ts.kernelspace_ts = ts_to_uint(*stamp);
         if(use_histograms)
           fill_histograms(&packet_ts, histograms);
       }
     }
   }
+
+  if (use_timestamps) {
+    clock_gettime(CLOCK_REALTIME, &ts);
+    packet_ts.userspace_exit_ts = ts_to_uint(ts);
+  }
+
   return packet_ts;
 }
 
 static void fill_histograms(packet_timestamps_t *packet_ts, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]) {
 
-  uint64_t user_space_time = packet_ts->enter_kernel - packet_ts->enter_user_space;
-  uint64_t kernel_space_time = packet_ts->leave_kernel - packet_ts->enter_kernel;
+  uint64_t user_space_time = packet_ts->userspace_exit_ts - packet_ts->userspace_enter_ts;
+  uint64_t kernel_space_time = packet_ts->userspace_enter_ts - packet_ts->kernelspace_ts;
 
   user_space_time /= 1000u;
   kernel_space_time /= 1000u;

packet-exchange/src/send_packet.c

@@ -166,7 +166,7 @@ packet_timestamps_t send_udp_packet(int use_etf, int use_timestamps,
 
   if (use_timestamps) {
     clock_gettime(CLOCK_REALTIME, &ts);
-    packet_ts.enter_user_space = ts_to_uint(ts);
+    packet_ts.userspace_enter_ts = ts_to_uint(ts);
   }
 
   memset(&sin, 0, sizeof(sin));
@@ -198,7 +198,7 @@ packet_timestamps_t send_udp_packet(int use_etf, int use_timestamps,
 
   if (use_timestamps) {
     clock_gettime(CLOCK_REALTIME, &ts);
-    packet_ts.enter_kernel = ts_to_uint(ts);
+    packet_ts.userspace_exit_ts = ts_to_uint(ts);
   }
 
   sendmsgerr = sendmsg(sock_fd, &msg, 0);
@@ -218,8 +218,8 @@ packet_timestamps_t send_udp_packet(int use_etf, int use_timestamps,
 
 static void fill_histograms(packet_timestamps_t *packet_ts, int64_t histograms[NB_HISTOGRAMS][MAX_HIST_VAL]) {
 
-  uint64_t user_space_time = packet_ts->enter_kernel - packet_ts->enter_user_space;
-  uint64_t kernel_space_time = packet_ts->leave_kernel - packet_ts->enter_kernel;
+  uint64_t user_space_time = packet_ts->userspace_exit_ts - packet_ts->userspace_enter_ts;
+  uint64_t kernel_space_time = packet_ts->kernelspace_ts - packet_ts->userspace_exit_ts;
 
   user_space_time /= 1000u;
   kernel_space_time /= 1000u;
@@ -267,7 +267,7 @@ static void process_timestamps(packet_timestamps_t *packet_ts, int64_t histogram
     if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SO_TIMESTAMPING) {
 
       struct timespec *stamp = (struct timespec *)CMSG_DATA(cmsg);
-      packet_ts->leave_kernel = ts_to_uint(*stamp);
+      packet_ts->kernelspace_ts = ts_to_uint(*stamp);
       fill_histograms(packet_ts, histograms);
 
     } else {

packet-exchange/src/server.c

@@ -226,13 +226,13 @@ int main(int argc, char *argv[]) {
         if(enable_timestamps) {
           printf("(%d)  Enter send_udp_packet timestamp: %" PRIu64 "\n",
                  stats->packets_received,
-                 stats->packet_ts.enter_user_space);
+                 stats->packet_ts.userspace_enter_ts);
           printf("(%d)  Call sendmsg timestamp         : %" PRIu64 "\n",
                  stats->packets_received,
-                 stats->packet_ts.enter_kernel);
+                 stats->packet_ts.userspace_exit_ts);
           printf("(%d)  Leave kernel timestamp         : %" PRIu64 "\n",
                  stats->packets_received,
-                 stats->packet_ts.leave_kernel);
+                 stats->packet_ts.kernelspace_ts);
         }
       }
     }

packet-exchange/src/utilities.h

@@ -14,9 +14,9 @@
 #define NB_HISTOGRAMS 3
 
 typedef struct packet_timestamps {
-  uint64_t enter_user_space;
-  uint64_t enter_kernel;
-  uint64_t leave_kernel;
+  uint64_t userspace_enter_ts;
+  uint64_t userspace_exit_ts;
+  uint64_t kernelspace_ts;
 } packet_timestamps_t;
 
 uint64_t ts_to_uint(struct timespec t);