Update documentation, improve histograms, fix mistakes

Fix gpio.h
Fix small mistake in print_hist function
Remove sudo command in run-cyclictest
Improve histograms to make them similar to cyclictest's ones

README.md

 # tsn-rt-measures repository
 
-This repository contains various programs I wrote to measure the TSN capabilities
-of three OLinuXino LIME2 boards I have.
+This repository contains various programs I wrote to do TSN measurements
 
-A recap of everything I have done and the current progress is available [here](https://www.nexedi.com/NXD-Johan.Huge.Internship.Progress)
+## clockres
 
-## cyclictest like programs
+clockres is used to determine the resolution of the clock (CLOCK_MONOTONIC), by doing successive
+calls to clock_gettime
+
+## packet-exchange
 
-clockres, and packet-exchange are C programs wrote using
-the same structure as cyclictest, and borrows large portions of the cyclictest
-code:   
+packet-exchange has a client and a server, which exchange UDP ethernet packets using real-time threads,
+and can send them on a ETF qdisc by setting a txtime timestamp. Software timestamps can be generated using
+the SO_TIMESTAMPING option to measure time spent in kernel.
+A Round-Trip Time option also exists to measure RTT of an UDP packet.
+XDP sockets can also be used to exchange packets
 
-They have a real-time thread doing measurements or time sensitive operations 
+The client and server programs have a structure similar to cyclictest
+They have a real-time thread doing measurements and time sensitive operations 
 at regular intervals, and a non real-time thread doing the I/O operations to
 report the results shared by the real-time thread.
-The real-time thread can be assigned to CPU1 (which I chose to isolate on the Olimex boards) and
-can have it's priority set as an option, default is to 99. It writes results in a static structure.
+The real-time thread can be assigned to a CPU and can have it's priority set as
+an option, default is to 98.
 
-* clockres is used to determine the resolution of the clock (CLOCK_MONOTONIC), by doing successive
-calls to clock_gettime
+## Scripts
 
-* packet-exchange has a client and a server, which exchange UDP ethernet packets using real-time threads,
-and can send them on a ETF qdisc by setting a txtime timestamp. software timestamps can be generated using
-the SO_TIMESTAMPING option, and timestamps inside the userspace program, for measuring purposes. A RTT 
-option also exists to measure RTT of an UDP packet.
+- run-server, run-client
+Front-end interface to the client and server programs from packet-exchange
+Uses create-qdisc to setup qdisc
+Can use trace-cmd for tracing
+Uses /etc/hosts to find IP address
 
-## Scripts
+- run-ptp4l, run-phc2sys, run-cyclictest
+Easier interfaces to use ptp4l, phc2sys and cyclictest
+
+- test-board-synchro
+This script can be used in a setup with two boards connected to one computer
+It will synchronize the computer and the boards with linux ptp, and have the computer send timestamps to the boards at a regular interval.
+On receiving these timestamps, the boards will toggle one of their GPIO pins at the given timestamp, and the signals can be measured with an oscilloscope, logic analyzer, microcontroller with input capture etc...
+
+- create-qdisc:
+Setups ETF or pfifo_fast qdisc, was written for the lime2 board whose
+NIC has only one queue, it should not be used for NIC with multiple RX queues
+
+- packet-histogram-stop, packet-histogram-stop
+Can automate measures between two boards using ssh and expect scripts
+(Needs to be updated: currently has no documentation, and might be broken)
+
+- sudossh
+Execute a command with root privileges on the olimex boards without having to enter the password
+
+- exec-ssh-nohup
+Execute a command and fork it so that it can still run after ssh is closed
+
+- txtime-stats.py
+Used for parsing for the tcpdump option in run-server
+
+- parse-saleae-measures.py
+Used to parse measures from the Saleaea logic analyzer
 
-The scripts folder contains various useful bash scripts to avoid
-typing long commands on the boards
 
 ## Measure analysis
 

packet-exchange/src/client.c

@@ -390,15 +390,19 @@ static void print_histograms() {
     max_hist_val = histogram_max(histogram, MAX_RTT - 1);
   }
 
+  printf("# Histogram\n");
+
+  for (int j = 0; j < max_hist_val; j++)
+    printf("%06d %015" PRIi64 "\n", j, histogram[j]);
+
   printf(
-      "Duration: %dh%d\n"
-      "High kernel latencies: %" PRIu64 "\n"
-      "Invalid parameters: %" PRIu64 "\n"
-      "Missed deadlines: %" PRIu64 "\n",
+      "# Duration: %dh%d\n"
+      "# High kernel latencies: %" PRIu64 "\n"
+      "# Invalid parameters: %" PRIu64 "\n"
+      "# Missed deadlines: %" PRIu64 "\n",
       duration_hour, duration_minutes, egress_stats.high_kernel_latency,
       egress_stats.invalid_parameter, egress_stats.missed_deadline);
 
-  for (int j = 0; j < max_hist_val; j++) printf("%" PRIi64 "\n", histogram[j]);
 }
 
 static void sighand(int sig_num) {

packet-exchange/src/common.c

@@ -25,7 +25,7 @@ struct timespec uint_to_ts(uint64_t t) {
 void add_ns(struct timespec *t, uint64_t ns) {
   t->tv_nsec += ns;
 
-  while (t->tv_nsec >= ((uint64_t) NSEC_PER_SEC)) {
+  while (t->tv_nsec >= ((int64_t) NSEC_PER_SEC)) {
     t->tv_sec += 1;
     t->tv_nsec -= NSEC_PER_SEC;
   }

packet-exchange/src/gpio.h

@@ -2,6 +2,6 @@
 #define GPIO_H
 
 void enable_gpio(int gpio_index);
-void toggle_gpio(int gpio_index);
+void toggle_gpio();
 
 #endif

packet-exchange/src/server.c

@@ -49,7 +49,6 @@ typedef struct main_params {
   int verbose;
   int enable_tracing;
   int enable_xdp_tracing;
-  int enable_graph;
 } main_param_t;
 
 static void process_options(int argc, char *argv[]);
@@ -95,7 +94,6 @@ static void help(char *argv[]) {
       "          POLL_MODE: 0 for polling, 1 for combination of both\n"
       "       -X                 Trace during XDP packet reception\n"
       "       -T THRESHOLD       Enable tracing until THRESHOLD is reached\n"
-      "       -G                 Enable function_graph tracer, used with -T\n"
       "       -v                 Verbose\n"
       "\n",
       argv[0]);
@@ -278,7 +276,6 @@ int main(int argc, char *argv[]) {
   main_params.refresh_rate = 50000;
   main_params.verbose = 0;
   main_params.enable_tracing = 0;
-  main_params.enable_graph = 0;
   enable_timestamps = 0;
   enable_histograms = 0;
   tsn_task = RECV_PACKET_TASK;
@@ -377,24 +374,30 @@ static void print_histograms() {
   if (enable_timestamps) {
     max_latency = histogram_max(kernel_latency_hist, MAX_KERNEL_LATENCY - 1);
 
+    printf("# Histogram\n");
+
+    for (int j = 0; j < max_latency; j++)
+      printf(" %06d %015" PRIi64 "\n", j, kernel_latency_hist[j]);
+
     printf(
-        "Duration: %dh%d\n"
-        "Lost packets: %" PRIu64 "\n",
+        "# Duration: %dh%d\n"
+        "# Lost packets: %" PRIu64 "\n",
         duration_hour, duration_minutes, ingress_stats.high_jitter);
 
-    for (int j = 0; j < max_latency; j++)
-      printf("%" PRIi64 "\n", kernel_latency_hist[j]);
   }
 
+  printf("# Histogram\n");
+
+  for (int j = min_jitter; j < max_jitter; j++)
+    printf(" %06d %015" PRIi64 "\n", j, jitter_hist[j]);
+
   printf(
-      "Middle: %d\n"
-      "Duration: %dh%d\n"
-      "Lost packets\": %" PRIu64 "\n",
+      "# Middle: %d\n"
+      "# Duration: %dh%d\n"
+      "# Lost packets: %" PRIu64 "\n",
       MAX_JITTER / 2 - min_jitter, duration_hour, duration_minutes,
       ingress_stats.high_jitter);
 
-  for (int j = min_jitter; j < max_jitter; j++)
-    printf("%" PRIi64 "\n", jitter_hist[j]);
 }
 
 static void sighand(int sig_num) {
@@ -479,9 +482,6 @@ static void process_options(int argc, char *argv[]) {
         main_params.enable_tracing = 1;
         thread_params.latency_threshold = atoi(optarg);
         break;
-      case 'G':
-        main_params.enable_graph = 1;
-        break;
     }
   }
 

scripts/multissh

-#!/bin/bash
-
-usage() {
-  cat << ENDUSAGE
-Usage: $0 CMD
-       $0 sudo CMD
-ENDUSAGE
-  exit 1;
-}
-
-if [ -z "$1" ]; then
-  usage
-fi
-
-if [ $1 == "sudo" ]; then
-  if [ -z "$2" ]; then
-    usage
-  fi
-  ./sudossh emerald "$2";
-  ./sudossh slate "$2";
-  ./sudossh onyx "$2";
-else
-  ssh emerald $1&
-  ssh onyx $1&
-  ssh slate $1
-fi

scripts/packet-histogram-start

 #!/bin/bash
 
 usage() {
-  echo "Usage: $0 OPTIONS -c CLIENT_BOARD SERVER_BOARD" 1>&2;
-  echo "       $0 OPTIONS -s [-t]  (-p | (-e DELTA -o ETF_OFFSET)) CLIENT_BOARD SERVER_BOARD" 1>&2;
-  echo "       $0 OPTIONS -C [-l] BOARD" 1>&2;
-  echo "OPTIONS: [-h] [-i INTERVAL] [-k KERNEL_VERSION]" 1>&2;
+  cat << ENDUSAGE
+Usage: $0 OPTIONS -c CLIENT_BOARD SERVER_BOARD" 1>&2;
+       $0 OPTIONS -s [-t]  (-p | (-e DELTA -o ETF_OFFSET)) CLIENT_BOARD SERVER_BOARD" 1>&2;
+       $0 OPTIONS -C [-l] BOARD" 1>&2;
+OPTIONS: [-h] [-i INTERVAL] [-k KERNEL_VERSION]" 1>&2;
+ENDUSAGE
+1>&2;
+
   exit 1;
 }
 

scripts/run-client

@@ -4,12 +4,40 @@ script_dir=$(dirname $(realpath $0))
 
 usage() {
   cat << ENDUSAGE
-Usage: $0 QDISC_OPT [CLIENT_OPTS] BOARD
-   QDISC_OPTS: (-e delta [-o etf_offset] -H | -p) [-q]
-   CLIENT_OPTS: -bgt -c DELAY -s NS -i INTERVAL -I if -a CPU -d TX_BUF_LEN [TRACE_OPTS]
-   TRACE_OPTS: [-T -P TRACER -E EVENTS -B SIZE]
-   default tracer opts: irq, sched, net_dev_start_xmit, 
-                        net_dev_xmit, net_dev_xmit_timeout
+Usage: $0 [-h] QDISC_OPT [CLIENT_OPTS] BOARD_HOSTNAME
+   -h              Show help
+   QDISC_OPTS: (-e DELTA [-o USEC] -H | -p) [-q]
+         Which qdisc to use (will call create-qdisc script)
+         -e DELTA       Use ETF (Earlier Txtime First) qdisc with specified delta
+                        (check tc-etf man page for more information)
+         -o USEC        Offset in userspace program to set the timestamp passed to ETF qdisc
+         -H             Use hardware offloading for the ETF qdisc (not all hardware supports this)
+         -p             Use pfifo_fast qdisc (default one)
+         -q             Don't setup qdiscs with create-qdisc script
+   CLIENT_OPTS:    -bgt (-c DELAY -s NS | -d TX_BUF_LEN) -i INTERVAL -I if -a CPU [TRACE_OPTS]
+         Options passed to the C client program (everything here is optional)
+         -b             Measure round trip time 
+         -g             Generate histogram of the measures
+         -t             Use SO_TIMESTAMPS to see how much time packet spent in kernel
+         -c DELAY       Send a timestamp in the packet with the specified delay (in us)
+                        (to be used with PTP)
+         -s NS          Specify a CLOCK_REALTIME timestamp at which client should start
+                        (to be used with PTP)
+         -d TX_BUF_LEN  Length of the data sent in the packets
+         -i USEC        Interval at which the RT thread should send packets
+         -I IF          Network interface to use to send the packets
+         -a CPU         CPU on which to pin the program
+   TRACE_OPTS:     -T [-P TRACER -E EVENTS -B SIZE]
+         Options to trace with trace-cmd until ETF deadline is missed
+         (see trace-cmd man page and ftrace documentation)
+         -T             Enable tracing
+         -P TRACER      Which trace to use when tracing: function, function_graph, wakeup etc...
+                        Default: function
+         -E EVENTS      Specify which events to trace (e.g: "-E "-e net -e irq")
+                        Default: irq, sched, net_dev_start_xmit,
+                                 net_dev_xmit, net_dev_xmit_timeout
+         -B SIZE        Size of the buffer for each CPU in kilobytes
+   BOARD_HOSTNAME /etc/hosts is used to find the IP of the given hostname
 ENDUSAGE
 1>&2;
 
@@ -112,10 +140,6 @@ qdisc_options+=" -I $interface"
 client_options+=" -a $cpu"
 
 board_name=$1
-
-if [ $board_name != "emerald" ] && [ $board_name != "onyx" ] && [ $board_name != "slate" ]; then
- usage
-fi 
 board_ip=$(cat /etc/hosts | grep $board_name | awk '{print $1}')
 
 if [ -z "${use_etf}" ] && [ -z "${use_pfast}" ]; then

scripts/run-cyclictest

 #!/bin/bash
 
 usage() {
-  echo "Usage: $0 -hgl -i INTERVAL" 1>&2;
+  cat << ENDUSAGE
+Usage: $0 -hgl -i INTERVAL" 1>&2
+   -h             Show help
+   -g             Generate histogram of the measures
+   -l             Generate artificial load with hackbench
+                  (This will cause heavy lag, program then needs to be stopped with
+                   killall run-cyclictest to prevent hackbench thread to keep
+                   being spawned)
+   -i INTERVAL    Interval passed to cyclictest (see cyclictest man page)
+ENDUSAGE
+1>&2;
+
   exit 1;
 }
 
@@ -40,7 +51,7 @@ if [ -n "$create_load" ]; then
 fi
 
 if [ -n "$generate_histogram" ]; then
-  sudo cyclictest $cyclictest_opts -h 400 -q > cyclictest_hist
+  cyclictest $cyclictest_opts -h 400 -q > cyclictest_hist
 else
-  sudo cyclictest $cyclictest_opts
+  cyclictest $cyclictest_opts
 fi

scripts/run-ptp4l

@@ -9,7 +9,18 @@ opts=""
 interface="eth0"
 
 usage() {
-  echo "Usage: $0 [-r -p telecom|gPTP|OTHER -s -H -i IF -o OPTS]" 1>&2;
+  cat << ENDUSAGE
+Usage: $0 [-h] [-r -p telecom|gPTP|OTHER -s -H -i IF -o OPTS]
+   -h              Show help
+   -r              Kill all ptp4l programs before
+   -p              Use a linuxptp profile (see in linuxptp/configs)
+   -s              Use slave mode
+   -H              Use hardware timestamping
+   -i IF           Set network interface to be used
+   -o OPTS         Pass additionnal options to linuxptp
+ENDUSAGE
+1>&2;
+
   exit 1;
 }
 

scripts/run-server

@@ -8,10 +8,34 @@ usage() {
 }
 usage() {
   cat << ENDUSAGE
-Usage: $0 [-I if] SERVER | TCPDUMP [TRACE_OPTS]
-   SERVER: [-bct] [(-x | -X) POLL] [-g INTERVAL] [-a CPU]
+Usage: $0 [-h] [-I if] [SERVER] | TCPDUMP [TRACE_OPTS]
+   -h              Show help
+   SERVER: -bct ((-x | -X) POLL) -g INTERVAL -a CPU
+         Options passed to the C server program (everything here is optional)
+         -b             Send back packets for round trip measurements
+         -c             Emit a signal on GPIO at the timestamp given in packet
+                        (to be used with -c option in client program)
+         -t             Use SO_TIMESTAMPS to see how much time packet spent in kernel
+         -x POLL        Use XDP sockets, with a global libbpf installation
+         -X POLL        Use XDP sockets, with libbpf located in \$HOME/libbpf folder
+            POLL: Polling mode used in server program, 0 to poll with poll function,
+                  1 to do active polling
+         -g USEC        Generate histograms for measures with the specified interval
+         -a CPU         CPU on which to pin the program
    TCPDUMP: -d NB_PACKETS [-i INTERVAL]
-   TRACE_OPTS: (-T LATENCY_THRESHOLD -G) | -E LATENCY_THRESHOLD
+         Will use tcpdump to capture the packets and measure jitter
+         -d NB_PACKETS  Use tcpdump to capture given amount of packets
+                        (don't set it too high, it doesn't scale well)
+         -i USEC        Specify which interval was used in the client
+                        Default: 1ms
+   TRACE_OPTS: -T LATENCY_THRESHOLD [-P TRACER -E XDP_EVENTS]
+         Trace until latency threshold is met
+         -T THRESHOLD   Enable tracing until threshold is met
+         -P TRACER      Which trace to use when tracing: function, function_graph, wakeup etc...
+                        Default: function
+         -E EVENTS      Specify which events to trace (e.g: "-E "-e net -e irq")
+                        Default: irq, sched, net, napi
+         -B SIZE        Size of the buffer for each CPU in kilobytes
 ENDUSAGE
 1>&2;
 
@@ -24,10 +48,11 @@ server_options="-p 99"
 make_opts=""
 ip="10.100.21."
 tcpdump_interval=1000000
-tracecmd_events="-e irq -e sched -e xdp -e net -e page_pool -e preemptirq -e napi"
+tracecmd_events="-e irq -e sched -e net -e napi"
+tracecmd_opts=""
 cpu=1
 
-while getopts "a:b:chtx:X:d:i:g:I:T:GE:" opt; do
+while getopts "a:b:chtx:X:d:i:g:I:T:E:P:B:" opt; do
   case "${opt}" in
     a )
         cpu=${OPTARG}
@@ -69,14 +94,17 @@ while getopts "a:b:chtx:X:d:i:g:I:T:GE:" opt; do
         server_options+=" -x ${OPTARG}"
 	make_opts=" -e WITH_GIT_XDP=1"
         ;;
-    T )
-        server_options+=" -T ${OPTARG}"
-      ;;
-    G )
-        server_options+=" -G"
+    B )
+        tracecmd_opts+=" -m ${OPTARG} -b ${OPTARG}"
       ;;
     E )
-        enable_xdp_events=1
+        tracecmd_events=${OPTARG}
+      ;;
+    P )
+        tracecmd_opts+=" -p ${OPTARG}"
+      ;;
+    T )
+        use_tracer=1
         server_options+=" -T ${OPTARG}"
       ;;
     * )
@@ -90,9 +118,6 @@ shift $((OPTIND-1))
 server_options+=" -f $interface -a $cpu"
 
 if [ -n "${use_rtt}" ]; then
-  if [ $board_name != "emerald" ] && [ $board_name != "onyx" ] && [ $board_name != "slate" ]; then
-   usage
-  fi 
   board_ip=$(cat /etc/hosts | grep $board_name | awk '{print $1}')
   server_options+=" -b $board_ip"
 fi
@@ -118,9 +143,9 @@ else
   echo "make $make_opts server";
   cd $script_dir/../packet-exchange/build;make $make_opts server;cd $script_dir
 
-  if [ -n "${enable_xdp_events}" ]; then
-    echo "trace-cmd record $tracecmd_events $script_dir/../packet-exchange/build/server $server_options";
-    trace-cmd record $tracecmd_events $script_dir/../packet-exchange/build/server $server_options;
+  if [ -n "${use_tracer}" ]; then
+    echo "trace-cmd record $tracecmd_opts $tracecmd_events $script_dir/../packet-exchange/build/server $server_options";
+    trace-cmd record $tracecmd_opts $tracecmd_events $script_dir/../packet-exchange/build/server $server_options;
   elif [ -z "${use_histogram}" ]; then
     echo "server $server_options";
     $script_dir/../packet-exchange/build/server $server_options;

scripts/test-board-synchro

 #!/bin/bash
 
 usage() {
-  echo "Usage: $0 [-i USEC -c USEC]" 1>&2;
+  cat << ENDUSAGE
+Usage: $0 [-i USEC -c USEC -s] BOARD1_HOSTNAME BOARD2_HOSTNAME
+   -h             Show help
+   -i USEC        Specify which interval to use in client
+   -c USEC        Specify which offset to use for the timestamp in the packet
+   -s             Use software timestampings
+   BOARD_HOSTNAME Uses /etc/hosts to find the IP address associated to the hostname
+ENDUSAGE
+1>&2;
+
   exit 1;
 }
 
 interval=4000000
 
-while getopts "c:i:o:r" opt; do
+while getopts "c:i:o:rs" opt; do
   case "${opt}" in
     i )
         interval=${OPTARG}
@@ -15,6 +24,9 @@ while getopts "c:i:o:r" opt; do
     c )
         delay=${OPTARG}
         ;;
+    s )
+        software_ts=1
+        ;;
     * )
         usage
         ;;
@@ -23,6 +35,15 @@ done
 
 shift $((OPTIND-1))
 
+if [ $# -ne 2 ]; then
+  usage
+fi
+
+board1=$1
+board2=$2
+
+exit
+
 # Setup irqs
 
 ksoftirq1_pid=$(ps -eL -o pid,cmd | grep "ksoftirqd/1" | awk '{ print $1 }' | head -n1)
@@ -37,15 +58,21 @@ chrt -f -p 97 $ksoftirq2_pid;
 echo "Starting ptp4l on master";
 
 killall ptp4l;
-./run-ptp4l -i enp1s0 -H;
-./run-ptp4l -i enp2s0 -H;
+
+if [ -n "$software_ts" ]; then
+  ./run-ptp4l -i enp1s0;
+  ./run-ptp4l -i enp2s0;
+else
+  ./run-ptp4l -i enp1s0 -H;
+  ./run-ptp4l -i enp2s0 -H;
+fi
 
 echo "Starting ptp4l on slaves";
 
-./sudossh slate "killall ptp4l";
-./sudossh onyx "killall ptp4l";
-./sudossh slate "run-ptp4l -s";
-./sudossh onyx "run-ptp4l -s";
+./sudossh $board1 "killall ptp4l";
+./sudossh $board2 "killall ptp4l";
+./sudossh $board1 "run-ptp4l -s";
+./sudossh $board2 "run-ptp4l -s";
 
 echo "Waiting for ptp to take effect...";
 
@@ -62,14 +89,14 @@ sleep 5;
 
 echo "Start servers on slave";
 
-./sudossh slate "setup_irqs";
-./sudossh onyx "setup_irqs";
+./sudossh $board1 "setup_irqs";
+./sudossh $board2 "setup_irqs";
 
-./sudossh slate "killall server";
-./sudossh onyx "killall server";
+./sudossh $board1 "killall server";
+./sudossh $board2 "killall server";
 
-./exec-ssh-nohup slate "run-server -c";
-./exec-ssh-nohup onyx "run-server -c";
+./exec-ssh-nohup $board1 "run-server -c";
+./exec-ssh-nohup $board2 "run-server -c";
 
 cd ../gettime/build;
 make;
@@ -88,5 +115,5 @@ if [ -z "$delay" ]; then
   delay=$((interval / 2))
 fi
 
-./run-client -a 1 -q -i $interval -p -I enp1s0 -c $delay -s $time slate &> client_enp1s0_log&
-./run-client -a 2 -q -i $interval -p -I enp2s0 -c $delay -s $time onyx &> client_enp2s0_log&
+./run-client -a 1 -q -i $interval -p -I enp1s0 -c $delay -s $time $board1 &> client_enp1s0_log&
+./run-client -a 2 -q -i $interval -p -I enp2s0 -c $delay -s $time $board2 &> client_enp2s0_log&