import numpy as np

progress_indicator = in_stream["Progress Indicator"]
in_data_stream = in_stream["Data Stream"]
start = progress_indicator.getIntOffsetIndex()
total_size = in_data_stream.getSize()

if start >= total_size:
  return

chunk_size = 1024 * 1024
end = min(start + chunk_size, total_size)

log_data = ''.join(in_data_stream.readChunkList(start, end))

# Some lines may contain several contiguous JSON objects
# Normalize to one JSON object per line
newlines_offset = log_data.count('}{')
log_data = log_data.replace('}{', '}\n{')

# Last chunk may contain an incomplete JSON object: leave it for next iteration
log_data_lines = log_data.splitlines()
if len(log_data_lines) < 2:
  return
log_data = '\n'.join(log_data_lines[:-1])

end = start + len(log_data) - newlines_offset + 1

log_data = log_data.decode('utf8')

e_rab_data_array = None
e_utran_data_array = None

for array in out_array:
  if array['variation'] == 'e_rab':
    e_rab_data_array = array['Data Array']
  if array['variation'] == 'e_utran':
    e_utran_data_array = array['Data Array']

try:
  vt, v_initial_epsb_estab_sr, v_added_epsb_estab_sr, evt, v_ip_throughput_qci = context.Base_getOrsKpiValues(log_data)
except AssertionError:
  # If some data is invalid: ignore data chunk and move on rather than crashing
  progress_indicator.setIntOffsetIndex(end)
  return

e_rab_dtype = np.dtype([
  ('vt', 'float'),
  ('vInitialEPSBEstabSR_lo', 'float64'),
  ('vInitialEPSBEstabSR_hi', 'float64'),
  ('vAddedEPSBEstabSR_lo', 'float64'),
  ('vAddedEPSBEstabSR_hi', 'float64'),
])

e_utran_dtype = np.dtype([
  ('evt', 'float'),
  ('qci', 'float'),
  ('dl_lo', 'float64'),
  ('dl_hi', 'float64'),
  ('ul_lo', 'float64'),
  ('ul_hi', 'float64'),
])

e_rab_array = e_rab_data_array.getArray()
if not e_rab_array:
  e_rab_array = e_rab_data_array.initArray(shape=(0,), dtype=e_rab_dtype)
e_rab_array_data = []

for i in range(len(vt)):
  e_rab_array_data.append((vt[i], v_initial_epsb_estab_sr[i][0], v_initial_epsb_estab_sr[i][1], v_added_epsb_estab_sr[i][0], v_added_epsb_estab_sr[i][1]))

if e_rab_array_data:
  e_rab_array_data =  np.ndarray((len(e_rab_array_data), ), e_rab_dtype, np.array(e_rab_array_data))
  e_rab_array.append(e_rab_array_data)

e_utran_array = e_utran_data_array.getArray()
if not e_utran_array:
  e_utran_array = e_utran_data_array.initArray(shape=(0,), dtype=e_utran_dtype)
e_utran_array_data = []
seen_qci_list = []
i = e_utran_array.shape[0] - 1
qci_idx = 1
while (i >= 0) and (e_utran_array[i][qci_idx] not in seen_qci_list):
  seen_qci_list.append(e_utran_array[i][qci_idx])
  i -= 1

for i in range(len(evt)):
  for qci_data in v_ip_throughput_qci[i]:
    qci = qci_data[0]
    qci_data_hi = [qci_data[2], qci_data[4]]
    if qci in seen_qci_list or any([data != 0 for data in qci_data_hi]):
      e_utran_array_data.append((evt[i], qci, qci_data[1], qci_data[2], qci_data[3], qci_data[4]))

if e_utran_array_data:
  e_utran_array_data =  np.ndarray((len(e_utran_array_data), ), e_utran_dtype, np.array(e_utran_array_data))
  e_utran_array.append(e_utran_array_data)

progress_indicator.setIntOffsetIndex(end)
return
