amari.xlog: Sync, reverse reading, timestamps for eNB < 2022-12-01

Rework XLog protocol to come with periodic sync events that come from time to
time so that xlog stream becomes self-synchronizing. Sync events should be
useful for Wendelin to start reading xlog stream from any point, and to verify
that the stream is ok by matching its content vs messages schedule coming in
the syncs.

Teach xlog.Reader to read streams in reverse order from end to start. This
should be useful to look at tail of a log without reading it in full from the
start.

Teach xlog.Reader to reconstruct messages timestamps for xlog streams produced
with Amarisoft releases < 2022-12-01. There messages do not have .utc field
added in https://support.amarisoft.com/issues/21934 and come with only .time
field that represent internal eNB time using clock originating at eNB startup.
We combine message.time and δ(utc, enb.time) from sync to build message.timestamp .

See individual patches for details and
kirr/xlte!3 for preliminary discussion.

/reviewed-by @xavier_thompson
/reviewed-on kirr/xlte!4

amari/__init__.py

@@ -63,7 +63,8 @@ def connect(ctx, wsuri):  # -> Conn
 class Conn:
     # .wsuri            websocket uri of the service
     # ._ws              websocket connection to service
-    # ._srv_ready_msg   message we got for "ready"
+    # .srv_ready_msg    message we got for "ready"
+    # .t_srv_ready_msg  timestamp of "ready" reception
 
     # ._mu              sync.Mutex
     # ._rxtab           {} msgid -> (request, rx channel)  | None
@@ -76,6 +77,7 @@ class Conn:
     def __init__(conn, ws, wsuri):
         try:
             msg0_raw = ws.recv()
+            t_msg0 = time.now()
             msg0 = json.loads(msg0_raw)
             # TODO also support 'authenticate'
             if msg0['message'] != 'ready':
@@ -86,7 +88,8 @@ class Conn:
 
         conn.wsuri = wsuri
         conn._ws = ws
-        conn._srv_ready_msg = msg0
+        conn.srv_ready_msg   = msg0
+        conn.t_srv_ready_msg = t_msg0
 
         conn._mu         = sync.Mutex()
         conn._rxtab      = {}
@@ -236,12 +239,12 @@ class Conn:
 
     @property
     def srv_type(conn):
-        return conn._srv_ready_msg['type']
+        return conn.srv_ready_msg['type']
 
     @property
     def srv_name(conn):
-        return conn._srv_ready_msg['name']
+        return conn.srv_ready_msg['name']
 
     @property
     def srv_version(conn):
-        return conn._srv_ready_msg['version']
+        return conn.srv_ready_msg['version']

amari/kpi.py

@@ -47,7 +47,7 @@ class LogMeasure:
     # ._rlog        IO reader for enb.log
     #
     # ._estats      \/ last xlog.Message with read stats result
-    #               \/ last xlog.Event | LogError
+    #               \/ last xlog.Event\sync | LogError
     #               \/ None
     # ._m           kpi.Measurement being prepared covering [_estats_prev, _estats) | None
     # ._m_next      kpi.Measurement being prepared covering [_estats, _estats_next) | None
@@ -153,6 +153,10 @@ def _read(logm):
         if x is None:
             x = LogError.EOF # represent EOF as LogError
 
+        # ignore sync events
+        if isinstance(x, xlog.SyncEvent):
+            continue
+
         # handle messages that update current Measurement
         if isinstance(x, xlog.Message):
             if x.message == "x.drb_stats":
@@ -162,7 +166,7 @@ def _read(logm):
                 continue    # ignore other messages
 
 
-        # it is an error, event or stats.
+        # it is an error, event\sync or stats.
         # if it is an event or stats -> finalize timestamp for _m_next.
         # start building next _m_next covering [x, x_next).
         # shift m <- ._m <- ._m_next <- (new Measurement | None for LogError)

amari/kpi_test.py

@@ -536,6 +536,35 @@ def test_LogMeasure_cc_wraparound():
     readok(4, 10)           # 4-5
 
 
+# verify that LogMeasure ignores syncs in xlog stream.
+@func
+def test_LogMeasure_sync():
+    t = tLogMeasure()
+    defer(t.close)
+    _ = t.expect1
+
+    cc = 'rrc_connection_request'
+    CC = 'RRC.ConnEstabAtt.sum'
+
+    t.xlog( jstats(1, {}) )
+    t.xlog( jstats(2, {cc: 4}) )
+    t.xlog( '{"meta": {"event": "sync", "time": 2.5, "state": "attached", "reason": "periodic", "generator": "xlog ws://localhost:9001 stats[]/30.0s"}}' )
+    t.xlog( jstats(3, {cc: 7}) )
+
+    def readok(τ, CC_value):
+        _('X.Tstart',   τ)
+        _('X.δT',       int(τ+1)-τ)
+        if CC_value is not None:
+            _(CC,       CC_value)
+        else:
+            t.expect_nodata()
+        t.read()
+
+    readok(0.02, None)      # attach-1
+    readok(1, 4)            # 1-2
+    readok(2, 3)            # 2-3  jumping over sync
+
+
 # jstats returns json-encoded stats message corresponding to counters dict.
 # τ goes directly to stats['utc'] as is.
 def jstats(τ, counters):  # -> str