Delete img-kernel-utils and update test-board-synchro

Delete img-kernel-utils as they it needed to be moved to different repo
Remove JSON formatting for histograms
Update gettime
Update print histograms function
Replace all CLOCK_TAI by CLOCK_REALTIME
Change send_tx_delay to uint64_t
Prevent past timestamp from being set to start in client
Use uint64_t for interval in client and server
Prevent continuing if clock_nanosleep fails in client
Change GPIO 49 to 86
Fix mistake in add_ns and make interval and delay an option in test-board-synchro
Remove useless overhead when writing to GPIO
Add script to parse saleae's csv exported measures

.gitignore

@@ -6,7 +6,7 @@ clockres/build/clockres
 clockres/build/clockres_arm
 packet-exchange/build/server
 packet-exchange/build/client
-ptptime/build/ptptime
+gettime/build/gettime
 
 scripts/packet-histogram_stop-options*
 

gettime/build/Makefile

-PROG = ptptime
+PROG = gettime
 SRCDIR = ../src
 
-SRCS  = main.c
+SRCS  = gettime.c
 
 OBJS = $(SRCS:%.c=%.o)
 

gettime/src/main.c → gettime/src/gettime.c

@@ -58,9 +58,9 @@ static void help(char *argv[]) {
 }
 
 /*
- * Real-time thread: Sends packets at a regular intervall
+ * Real-time thread
  */
-static void *packet_sending_thread(void *p) {
+static void *rt_thread(void *p) {
   (void)p;
   struct timespec next, ptptime_ts;
   cpu_set_t mask;
@@ -73,7 +73,7 @@ static void *packet_sending_thread(void *p) {
 
   clock_gettime(CLOCK_MONOTONIC, &next);
 
-  // Packet sending loop
+  // Time lookup loop
   for (;;) {
     clock_gettime(clock_id, &ptptime_ts);
     ptptime = ts_to_uint(ptptime_ts);
@@ -144,8 +144,8 @@ int main(int argc, char *argv[]) {
   }
 
   // Create the real time thread
-  if (pthread_create(&thread, &attr, packet_sending_thread, NULL))
-    error(EXIT_FAILURE, errno, "Couldn't create packet sending thread");
+  if (pthread_create(&thread, &attr, rt_thread, NULL))
+    error(EXIT_FAILURE, errno, "Couldn't create rt thread");
 
   // Verbose loop
   for (;;) {

packet-exchange/src/client.c

@@ -37,11 +37,11 @@ typedef struct rtt_stat {
 } rtt_stat_t;
 
 typedef struct thread_param {
-  int interval;
+  uint64_t interval;
   unsigned int max_cycles;
   int priority;
   int etf_offset;
-  int send_tx_delay;
+  uint64_t send_tx_delay;
   int affinity_cpu;
   uint64_t start_ts;
 
@@ -116,6 +116,7 @@ static void *packet_sending_thread(void *p) {
   (void)p;
   struct timespec next, current, previous;
   uint64_t next_txtime = 0, tx_data = 0;
+  int ret;
   cpu_set_t mask;
 
   // Set thread CPU affinity
@@ -128,11 +129,19 @@ static void *packet_sending_thread(void *p) {
   }
 
   clock_gettime(CLOCK_MONOTONIC, &measures_start);
+  clock_gettime(CLOCK_REALTIME, &next);
 
-  if(thread_params.start_ts)
+  if (thread_params.start_ts) {
+    if(thread_params.start_ts < ts_to_uint(next)) {
+      fprintf(stderr, "start timestamp is in the past, aborting...\n");
+      exit(EXIT_FAILURE);
+    }
+    if(thread_params.start_ts > (ts_to_uint(next) + UINT64_C(3600000000000))) {
+      fprintf(stderr, "start timestamp is too high, aborting...\n");
+      exit(EXIT_FAILURE);
+    }
     next = uint_to_ts(thread_params.start_ts);
-  else
-    clock_gettime(CLOCK_REALTIME, &next);
+  }
 
   // Set txtime and TX data timestamp if required
   if (enable_etf || thread_params.send_tx_delay) {
@@ -178,7 +187,13 @@ static void *packet_sending_thread(void *p) {
     }
 
     previous = current;
-    clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, &next, NULL);
+
+    ret = clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, &next, NULL);
+    if(ret) {
+      fprintf(stderr, "clock_nanosleep returned error: %d, aborting...\n", ret);
+      exit(EXIT_FAILURE);
+    }
+
   }
 
   pthread_exit(NULL);
@@ -283,10 +298,8 @@ int main(int argc, char *argv[]) {
     usleep(main_params.refresh_rate);
 
     if (main_params.verbose) {
-
       // RTT stats printing
       if (tsn_task == RTT_TASK) {
-
         // N_CYCLES, RTT min / avg / max
         printf("%9" PRIu64 ": RTT: %4d %4d %4d\n", nb_cycles, rtt_stats.min_rtt,
                rtt_stats.avg_rtt, rtt_stats.max_rtt);
@@ -294,7 +307,6 @@ int main(int argc, char *argv[]) {
 
         // Packet send stats printing
       } else {
-
         // N_CYCLES, error queue, measured send interval min / avg / max
         printf("%9" PRIu64 ": [%4d, %4d], I (10us): %3d %3d %3d", nb_cycles,
                (int)egress_stats.invalid_parameter,
@@ -356,11 +368,11 @@ static void do_tsn_task(char *data, uint64_t next_txtime) {
   }
 }
 
-/* Print histograms in .json format
+/* Print histogram
  */
 static void print_histograms() {
   uint64_t duration;
-  int duration_hour, duration_minutes, interval;
+  int duration_hour, duration_minutes;
   int max_hist_val;
   uint64_t *histogram;
 
@@ -370,21 +382,6 @@ static void print_histograms() {
   duration_hour = duration / NSEC_PER_SEC / 3600;
   duration_minutes = duration / NSEC_PER_SEC / 60 - duration_hour * 60;
 
-  interval = thread_params.interval / 1000;
-
-  printf(
-      "{\"measure_sets\": [{"
-      "\"measure_type\": \"%s\","
-      "\"props_names\": [\"%s\"],"
-      "\"units\": [\"us\"],"
-      "\"props_type\": \"histogram\","
-      "\"metadata\": {\"i\": \"%dus\", \"duration\": \"%dh%d\", "
-      "\"etf_offset\": \"%dus\"},"
-      "\"props\": [[",
-      (tsn_task == SEND_PACKET_TASK) ? "packet_send_timestamps" : "packet_rtt",
-      (tsn_task == SEND_PACKET_TASK) ? "kernel_space" : "rtt", interval,
-      duration_hour, duration_minutes, thread_params.etf_offset);
-
   if (tsn_task == SEND_PACKET_TASK) {
     histogram = kernel_latency_hist;
     max_hist_val = histogram_max(histogram, MAX_KERNEL_LATENCY - 1);
@@ -393,9 +390,15 @@ static void print_histograms() {
     max_hist_val = histogram_max(histogram, MAX_RTT - 1);
   }
 
-  for (int j = 0; j < max_hist_val; j++)
-    printf("%" PRIi64 "%s", histogram[j], (j + 1 < max_hist_val ? ", " : ""));
-  printf("]]}]}\n");
+  printf(
+      "Duration: %dh%d\n"
+      "High kernel latencies: %" PRIu64 "\n"
+      "Invalid parameters: %" PRIu64 "\n"
+      "Missed deadlines: %" PRIu64 "\n",
+      duration_hour, duration_minutes, egress_stats.high_kernel_latency,
+      egress_stats.invalid_parameter, egress_stats.missed_deadline);
+
+  for (int j = 0; j < max_hist_val; j++) printf("%" PRIi64 "\n", histogram[j]);
 }
 
 static void sighand(int sig_num) {
@@ -407,7 +410,6 @@ static void sighand(int sig_num) {
 /* Process bash options
  */
 static void process_options(int argc, char *argv[]) {
-
   for (;;) {
     int c = getopt(argc, argv, "a:bc:d:e:ghi:l:p:q:r:s:tvT");
 
@@ -421,7 +423,7 @@ static void process_options(int argc, char *argv[]) {
         tsn_task = RTT_TASK;
         break;
       case 'c':
-        thread_params.send_tx_delay = atoi(optarg) * 1000;
+        thread_params.send_tx_delay = strtoull(optarg, NULL, 10) * 1000;
         break;
       case 'd':
         egress_params.tx_buffer_len = atoi(optarg);
@@ -442,7 +444,7 @@ static void process_options(int argc, char *argv[]) {
         exit(EXIT_SUCCESS);
         break;
       case 'i':
-        thread_params.interval = atoi(optarg) * 1000;
+        thread_params.interval = strtoull(optarg, NULL, 10) * 1000;
         break;
       case 'l':
         thread_params.max_cycles = atoi(optarg);

packet-exchange/src/common.c

@@ -25,7 +25,7 @@ struct timespec uint_to_ts(uint64_t t) {
 void add_ns(struct timespec *t, uint64_t ns) {
   t->tv_nsec += ns;
 
-  while ((unsigned int)t->tv_nsec >= NSEC_PER_SEC) {
+  while (t->tv_nsec >= ((uint64_t) NSEC_PER_SEC)) {
     t->tv_sec += 1;
     t->tv_nsec -= NSEC_PER_SEC;
   }

packet-exchange/src/gpio.c

 #include "gpio.h"
 #include <stdlib.h>
 #include <stdio.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <fcntl.h>
 
 static int gpio_state;
 static char cmd[128];
+static char * one = "1";
+static char * zero = "0";
+static int fd;
 
 void enable_gpio(int gpio_index) {
   sprintf(cmd, "echo %d > /sys/class/gpio/export", gpio_index);
   system(cmd);
   sprintf(cmd, "echo out > /sys/class/gpio/gpio%d/direction", gpio_index);
   system(cmd);
+
+  sprintf(cmd, "/sys/class/gpio/gpio%d/value", gpio_index);
+  fd = open(cmd, O_WRONLY);
 }
-void toggle_gpio(int gpio_index) {
-  sprintf(cmd, "echo %d > /sys/class/gpio/gpio%d/value", gpio_state, gpio_index);
-  system(cmd);
+
+void toggle_gpio() {
+
+  if(gpio_state)
+        write(fd, one, 1);
+  else
+        write(fd, zero, 1);
 
   gpio_state = !gpio_state;
 }

packet-exchange/src/recv_packet.h

@@ -8,7 +8,7 @@ typedef struct ingress_param {
 
 	int use_timestamps;
 	int xdp_polling_mode;
-	int interval;
+	uint64_t interval;
 
 	size_t tx_buffer_len;
 

packet-exchange/src/send_packet.c

@@ -94,7 +94,7 @@ void init_udp_send(egress_param_t *_params, egress_stat_t *_stats,
     error(EXIT_FAILURE, errno, "setsockopt SO_BINDTODEVICE failed\n");
 
   if (params->use_etf) {
-    sk_txtime.clockid = CLOCK_TAI;
+    sk_txtime.clockid = CLOCK_REALTIME;
     sk_txtime.flags = SOF_TXTIME_REPORT_ERRORS;
 
     if (setsockopt(sock_fd, SOL_SOCKET, SO_TXTIME, &sk_txtime,

packet-exchange/src/server.c

@@ -25,17 +25,17 @@
 #include <unistd.h>
 
 #include "common.h"
+#include "gpio.h"
 #include "recv_packet.h"
 #include "send_packet.h"
 #include "tracer.h"
-#include "gpio.h"
 
 // Structs
 
 enum TSNTask { RECV_PACKET_TASK, RTT_TASK, XDP_TASK };
 
 typedef struct thread_param {
-  int interval;
+  uint64_t interval;
   int priority;
   int affinity_cpu;
   uint64_t latency_threshold;
@@ -117,13 +117,10 @@ static void *emit_signal_thread(void *p) {
   pthread_mutex_lock(&emit_signal_mutex);
 
   for (;;) {
-
     pthread_cond_wait(&emit_signal_ts_received, &emit_signal_mutex);
-    clock_nanosleep(CLOCK_TAI, TIMER_ABSTIME, &emit_signal_next, NULL);
-
-    toggle_gpio(49);
-    printf("Toggled GPIO\n");
+    clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, &emit_signal_next, NULL);
 
+    toggle_gpio();
   }
 
   pthread_mutex_unlock(&emit_signal_mutex);
@@ -144,7 +141,8 @@ static void *tsn_thread(void *p) {
     CPU_ZERO(&mask);
     CPU_SET(thread_params.affinity_cpu, &mask);
     if (sched_setaffinity(0, sizeof(mask), &mask))
-      error(EXIT_FAILURE, errno, "Could not set CPU affinity to CPU %d\n", thread_params.affinity_cpu);
+      error(EXIT_FAILURE, errno, "Could not set CPU affinity to CPU %d\n",
+            thread_params.affinity_cpu);
   }
 
   clock_gettime(CLOCK_MONOTONIC, &measures_start);
@@ -171,7 +169,7 @@ static void *tsn_thread(void *p) {
 
       // Emit signal
       if (thread_params.emit_signal) {
-        uint64_t emit_signal_t = *((uint64_t *) ingress_stats.data);
+        uint64_t emit_signal_t = *((uint64_t *)ingress_stats.data);
 
         pthread_mutex_lock(&emit_signal_mutex);
         emit_signal_next = uint_to_ts(emit_signal_t);
@@ -222,8 +220,7 @@ static void *tsn_thread(void *p) {
   return NULL;
 }
 
-static void create_thread(void * (*thread_function)(void *)) {
-
+static void create_thread(void *(*thread_function)(void *)) {
   pthread_t thread;
   pthread_attr_t attr;
   struct sched_param param;
@@ -257,7 +254,6 @@ static void create_thread(void * (*thread_function)(void *)) {
   // Create the real time thread
   if (pthread_create(&thread, &attr, thread_function, NULL))
     error(EXIT_FAILURE, errno, "Couldn't create packet receiving thread");
-
 }
 
 /* Main thread:
@@ -265,7 +261,6 @@ static void create_thread(void * (*thread_function)(void *)) {
  *   - Handles the IO and creates real time threads
  */
 int main(int argc, char *argv[]) {
-
   // Initial values
   ingress_stats.min_interval = INT_MAX;
   ingress_stats.avg_interval = 0;
@@ -318,11 +313,11 @@ int main(int argc, char *argv[]) {
   if (tsn_task == RTT_TASK)
     init_udp_send(&egress_params, &egress_stats, 0, 0, NULL);
 
-  if(thread_params.emit_signal) {
+  if (thread_params.emit_signal) {
     pthread_mutex_init(&emit_signal_mutex, NULL);
     pthread_cond_init(&emit_signal_ts_received, NULL);
 
-    enable_gpio(49);
+    enable_gpio(86);
   }
 
   create_thread(tsn_thread);
@@ -367,7 +362,7 @@ int main(int argc, char *argv[]) {
  */
 static void print_histograms() {
   uint64_t duration;
-  int duration_hour, duration_minutes, interval;
+  int duration_hour, duration_minutes;
   int max_latency, max_jitter, min_jitter;
 
   clock_gettime(CLOCK_MONOTONIC, &measures_end);
@@ -376,49 +371,30 @@ static void print_histograms() {
   duration_hour = duration / NSEC_PER_SEC / 3600;
   duration_minutes = duration / NSEC_PER_SEC / 60 - duration_hour * 60;
 
-  interval = thread_params.interval / 1000;
+  max_jitter = histogram_max(jitter_hist, MAX_JITTER - 1);
+  min_jitter = histogram_min(jitter_hist, MAX_JITTER - 1);
 
   if (enable_timestamps) {
-    printf(
-        "{\"measure_sets\": [{"
-        "\"measure_type\": \"packet_recv_timestamps\","
-        "\"props_names\": [\"kernel_space\"],"
-        "\"units\": [\"us\"],"
-        "\"props_type\": \"histogram\","
-        "\"metadata\": {\"i\": \"%dus\", \"duration\": \"%dh%d\","
-        "\"lost_packets\": \"%d\"},"
-        "\"props\": [[",
-        interval, duration_hour, duration_minutes,
-        ((int)ingress_stats.high_jitter));
-
     max_latency = histogram_max(kernel_latency_hist, MAX_KERNEL_LATENCY - 1);
 
+    printf(
+        "Duration: %dh%d\n"
+        "Lost packets: %" PRIu64 "\n",
+        duration_hour, duration_minutes, ingress_stats.high_jitter);
+
     for (int j = 0; j < max_latency; j++)
-      printf("%" PRIi64 "%s", kernel_latency_hist[j],
-             (j + 1 < max_latency ? ", " : ""));
-    printf("]]");
+      printf("%" PRIi64 "\n", kernel_latency_hist[j]);
   }
 
-  max_jitter = histogram_max(jitter_hist, MAX_JITTER - 1);
-  min_jitter = histogram_min(jitter_hist, MAX_JITTER - 1);
-
-  printf("%s", enable_timestamps ? "}, {" : "{\"measure_sets\": [{");
-
   printf(
-      "\"measure_type\": \"packet_jitter\","
-      "\"props_names\": [\"jitter\"],"
-      "\"units\": [\"us\"],"
-      "\"props_type\": \"histogram\","
-      "\"middle\": \"%d\","
-      "\"metadata\": {\"i\": \"%dus\", \"duration\": \"%dh%d\","
-      "\"lost_packets\": \"%d\"},"
-      "\"props\": [[",
-      MAX_JITTER / 2 - min_jitter, interval, duration_hour, duration_minutes,
-      ((int)ingress_stats.high_jitter));
+      "Middle: %d\n"
+      "Duration: %dh%d\n"
+      "Lost packets\": %" PRIu64 "\n",
+      MAX_JITTER / 2 - min_jitter, duration_hour, duration_minutes,
+      ingress_stats.high_jitter);
 
   for (int j = min_jitter; j < max_jitter; j++)
-    printf("%" PRIi64 "%s", jitter_hist[j], (j + 1 < max_jitter ? ", " : ""));
-  printf("]]}]}\n");
+    printf("%" PRIi64 "\n", jitter_hist[j]);
 }
 
 static void sighand(int sig_num) {
@@ -475,7 +451,7 @@ static void process_options(int argc, char *argv[]) {
         exit(EXIT_SUCCESS);
         break;
       case 'i':
-        thread_params.interval = atoi(optarg) * 1000;
+        thread_params.interval = strtoull(optarg, NULL, 10) * 1000;
         break;
       case 'g':
         enable_histograms = 1;

scripts/parse-saleae-measures.py

+#!/usr/bin/python
+
+import sys
+
+def line_to_ts(l):
+    ts_float = float(l.split(",")[0])
+    ts_ns = int(ts_float * 10**9)
+
+    return ts_ns
+
+if len(sys.argv) < 2:
+    print("You need to specify the csv file")
+    exit()
+
+filename=sys.argv[1]
+
+with open(filename, "r") as f:
+    lines = [l for l in f]
+    lines = lines[1:]
+    values = [ line_to_ts(l) for l in lines]
+    deltas = []
+
+    start = 1
+    if (values[1] - values[0]) > (values[2] - values[1]):
+        start = 2
+
+    for i in range(start, len(values), 2):
+
+        if (i+1) < len(values):
+            if (values[i] - values[i - 1]) > (values[i + 1] - values[i]):
+                print("Instruction missed at line {} ?".format(i))
+                exit()
+        deltas.append(values[i] - values[i-1])
+
+    min_d = min(deltas)
+    max_d = max(deltas)
+    avg_d = sum(deltas) / len(deltas)
+
+    for i in range(len(deltas)):
+        if deltas[i] == max_d:
+            print("Maximum delta at line {}".format(i))
+
+    min_d = min_d / 1000
+    max_d = max_d / 1000
+    avg_d = avg_d / 1000
+
+    print("Min: {}us\n"
+          "Max: {}us\n"
+          "Avg: {}us".format(min_d,
+                             max_d,
+                             avg_d))

scripts/test-board-synchro

 #!/bin/bash
 
 usage() {
-  echo "Usage: $0 [-r -i IF -o OPTS]" 1>&2;
+  echo "Usage: $0 [-i USEC -c USEC]" 1>&2;
   exit 1;
 }
 
-while getopts "i:o:r" opt; do
+interval=4000000
+
+while getopts "c:i:o:r" opt; do
   case "${opt}" in
     i )
-        interface=${OPTARG}
-        ;;
-    o )
-        opts+=${OPTARG}
+        interval=${OPTARG}
         ;;
-    r )
-        reset=1
+    c )
+        delay=${OPTARG}
         ;;
     * )
         usage
@@ -50,7 +49,7 @@ echo "Starting ptp4l on slaves";
 
 echo "Waiting for ptp to take effect...";
 
-sleep 10;
+sleep 35;
 
 # Run phc2sys
 
@@ -59,7 +58,7 @@ sleep 10;
 
 echo "Waiting for phc2sys to take effect...";
 
-sleep 2;
+sleep 5;
 
 echo "Start servers on slave";
 
@@ -79,11 +78,15 @@ time=$(./gettime -r -q)
 
 echo "CLOCK_REALTIME time is $time";
 
-time+=15000000000
+time=$((time + 10000000000))
 
 echo "Start clients on master";
 
 cd ../../scripts;
 
-./run-client -a 1 -q -i 1000000 -p -I enp1s0 -c 500000 -s $time slate &> client_enp1s0_log&
-./run-client -a 2 -q -i 1000000 -p -I enp2s0 -c 500000 -s $time onyx &> client_enp2s0_log&
+if [ -z "$delay" ]; then
+  delay=$((interval / 2))
+fi
+
+./run-client -a 1 -q -i $interval -p -I enp1s0 -c $delay -s $time slate &> client_enp1s0_log&
+./run-client -a 2 -q -i $interval -p -I enp2s0 -c $delay -s $time onyx &> client_enp2s0_log&