##############################################################################
#
# Copyright (c) 2002-2024 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
##############################################################################
from cStringIO import StringIO
from httplib import NO_CONTENT
import json
import msgpack
import random
import string

from Products.ERP5Type.tests.SecurityTestCase import SecurityTestCase


def generateRandomString(length=24, only_digits=False, hexadecimal=False):
  character_list = string.digits
  if not only_digits:
    if hexadecimal:
      character_list += 'ABCDEF'
    else:
      character_list += string.ascii_letters
  return ''.join([random.choice(character_list) for _ in xrange(length)])

class WendelinTelecomTest(SecurityTestCase):
  """
  Wendelin Telecom Test
  """

  def getTitle(self):
    return "Wendelin Telecom Test"

  def afterSetUp(self):
    # Set up variables for ORS ingestion testing
    self.ors_enb_log_ingestion = self.portal.portal_ingestion_policies.ors_enb_log_ingestion
    self.test_ors_example_log_valid = {
      'log': self.portal.web_page_module.test_example_ors_enb_log_valid.getTextContent()
    }
    self.test_ors_example_log_invalid = {
      'log': self.portal.web_page_module.test_example_ors_enb_log_invalid.getTextContent()
    }
    self.test_ors_example_log_empty = {'log': ""}

    # Set up ingestor user for performing ingestions
    self.ingestor_user = self.createWendelinTelecomUser('test_ingestor_%s' % generateRandomString(), None, 'ingestor')

  def beforeTearDown(self):
    self.abort()

    # Clean up all test items
    for module, portal_type in (
      (self.portal.project_module, 'Project'),
      (self.portal.person_module, 'Person'),
      (self.portal.data_acquisition_unit_module, 'Data Acquisition Unit'),
      (self.portal.data_supply_module, 'Data Supply'),
      (self.portal.data_stream_module, 'Data Stream'),
      (self.portal.data_ingestion_module, 'Data Ingestion'),
      (self.portal.data_analysis_module, 'Data Analysis'),
      (self.portal.data_array_module, 'Data Array'),
    ):
      object_list = module.objectValues(portal_type=portal_type)
      if object_list:
        test_object_id_list = [
          obj.getId() for obj in object_list \
          if ('test' in obj.getReference().lower() and 'default' not in obj.getId())
        ]
        if test_object_id_list:
          module.manage_delObjects(ids=test_object_id_list)
    self.tic()

  def createWendelinTelecomUser(self, reference, project, function):
    # Create and validate a new Person with an assignment linked to the provided project and function
    # Also generate and validate an ERP5 login for the Person
    user = self.portal.person_module.newContent(
      portal_type='Person',
      reference=reference
    )
    user.newContent(portal_type='Assignment', destination_project=project, function=function).open()
    user.newContent(portal_type='ERP5 Login', reference=reference, password=reference).validate()
    user.validate()
    self.tic()

    return user

  def registerOrs(self, tag_hostname_seed=None, tag_comp_id_seed=None, tag_enb_id_seed=None):
    # Create a Data Acquisition Unit and related Data Supply with a tag constructed from the provided seeds.
    # If any seed is NOT defined, it is generated at random.
    if tag_hostname_seed is None:
      tag_hostname_seed = generateRandomString(length=3, only_digits=True)
    if tag_comp_id_seed is None:
      tag_comp_id_seed = generateRandomString(length=4, only_digits=True)
    if tag_enb_id_seed is None:
      tag_enb_id_seed = generateRandomString(length=5, hexadecimal=True)

    ors_tag = 'ors%s' % tag_hostname_seed
    # Only include a subsequent section if a non-empty string is explicitly provided for it
    if tag_comp_id_seed != '':
      ors_tag += '_COMP-%s' % tag_comp_id_seed
    if tag_enb_id_seed != '':
      ors_tag += '_e0x%s' % tag_enb_id_seed
    ors_tag += 'Test'

    response = self.portal.ERP5Site_registerOrs(ors_tag)
    self.tic()

    # Fetch created items from the catalog
    data_acquisition_unit = self.portal.portal_catalog.getResultValue(
      portal_type='Data Acquisition Unit',
      reference=ors_tag,
      validation_state='validated'
    )
    data_supply = None
    if data_acquisition_unit is not None:
      data_supply = data_acquisition_unit.DataAcquisitionUnit_createOrsDataSupply(batch=1)

    # Return all associated items
    return {
      'response': response,
      'data_acquisition_unit': data_acquisition_unit,
      'data_supply': data_supply
    }

  def registerOrsClientProject(self, reference_seed=None, client_user_reference_seed=None):
    # Create a client project with the provided reference seed,
    # as well as a related ERP5 Person with the same reference seed, and an ERP5 login.
    # If NOT defined, reference_seed is generated at random.
    # If defined, client_user_reference_seed overrides reference_seed for the client user.
    if reference_seed is None:
      reference_seed = generateRandomString()
    project_reference = 'test_project_%s' % reference_seed
    project_title = 'Test Project %s' % reference_seed
    client_email = 'test_user_%s@test.wendelin-tele.com' % (client_user_reference_seed or reference_seed)
    client_user_reference = client_email.split('@')[0]

    # Call the script responsible for creating the project and the associated user, and store the JSON response
    response = self.portal.ProjectModule_registerOrsClientProject(
      project_reference,
      project_title,
      client_email,
      form_id='testing'
    )
    self.tic()

    # Fetch created items from the catalog
    project = self.portal.portal_catalog.getResultValue(
      portal_type='Project',
      reference=project_reference,
      title=project_title,
      validation_state='validated'
    )
    client_user = self.portal.portal_catalog.getResultValue(
      portal_type='Person',
      reference=client_user_reference,
      default_email_text=client_email,
      validation_state='validated'
    )

    # Return all associated items
    return {
      'response': response,
      'project': project,
      'client_user': client_user
    }

  def ingestOrsLogDataFromFluentd(self, log_data, ors_tag):
    # Simulate a fluentd instance sending the provided log data to Wendelin for ingestion
    reference = 'ors.%s' % ors_tag
    body = msgpack.packb([0, log_data], use_bin_type=True)
    env = {'CONTENT_TYPE': 'application/octet-stream'}
    path = self.ors_enb_log_ingestion.getPath() + '/ingest?reference=' + reference
    publish_kw = dict(
      env=env,
      user=self.ingestor_user.Person_getUserId(),
      request_method='POST',
      stdin=StringIO(body)
    )
    return self.publish(path, **publish_kw)

  def getDataStream(self, data_acquisition_unit):
    # Retrieve a Data Stream linked to the provided Data Acquisition Unit
    for line in data_acquisition_unit.getAggregateRelatedValueList(portal_type='Data Ingestion Line'):
      data_stream = line.getAggregateValue(portal_type='Data Stream')
      if data_stream:
        return data_stream

  def getDataAnalysis(self, data_supply):
    # Retrieve a Data Analysis linked to the provided Data Supply
    for data_analysis in data_supply.getSpecialiseRelatedValueList(portal_type='Data Analysis'):
      if data_analysis:
        return data_analysis

  def getDataArrays(self, data_analysis):
    # Retrieve the Data Arrays linked to the provided Data Analysis
    data_array_list = []
    for line in data_analysis.contentValues(portal_type='Data Analysis Line'):
      data_array = line.getAggregateValue(portal_type='Data Array')
      if data_array:
        data_array_list.append(data_array)
    return data_array_list

  def getOrsLogIngestionItems(self, log_data, reference, stop_data_analysis=False):
    # Simulate an ingestion of the provided log data
    response = self.ingestOrsLogDataFromFluentd(log_data, reference)
    self.tic()

    # Retrieve all items linked to the ingestion
    data_acquisition_unit = self.portal.portal_catalog.getResultValue(
      portal_type='Data Acquisition Unit',
      reference=reference
    )
    data_supply = data_acquisition_unit.DataAcquisitionUnit_createOrsDataSupply(batch=1)
    self.tic()

    data_stream = self.getDataStream(data_acquisition_unit)

    # Manually call the data analysis alarms to process the data into KPIs ASAP
    self.portal.portal_alarms.wendelin_handle_analysis.activeSense()
    self.tic()
    self.portal.portal_alarms.wendelin_handle_analysis.activeSense()
    self.tic()

    data_analysis = self.getDataAnalysis(data_supply)
    if stop_data_analysis and data_analysis.getSimulationState() == 'started':
      data_analysis.stop()
      self.tic()

    data_array_list = self.getDataArrays(data_analysis)

    # Return all associated items
    return {
      'response': response,
      'data_acquisition_unit': data_acquisition_unit,
      'data_supply': data_supply,
      'data_stream': data_stream,
      'data_analysis': data_analysis,
      'data_array_list': data_array_list
    }

  def checkDocumentPermissions(self, user, document, user_can_view, user_can_modify, user_can_add):
    user_id = user.Person_getUserId()

    # Check if the provided user can access and view the document
    if user_can_view:
      self.assertUserCanAccessDocument(user_id, document)
      self.assertUserCanViewDocument(user_id, document)
    else:
      self.failIfUserCanAccessDocument(user_id, document)
      self.failIfUserCanViewDocument(user_id, document)

    # Check if the provided user can edit the document
    if user_can_modify:
      self.assertUserCanModifyDocument(user_id, document)
    else:
      self.failIfUserCanModifyDocument(user_id, document)

    # Check if the provided user can add a document
    if user_can_add:
      self.assertUserCanAddDocument(user_id, document)
    else:
      self.failIfUserCanAddDocument(user_id, document)

  def checkIngestionDocumentsPermissions(self, user, ingestion_item_dict):
    user_destination_project = None
    user_function_list = None
    for assignment in user.contentValues(portal_type='Assignment'):
      if assignment.getValidationState() == 'open':
        user_destination_project = assignment.getDestinationProject()
        user_function_list = assignment.getFunctionList()

    ors_destination_project = ingestion_item_dict['data_supply'].getDestinationProject()
    same_project = (user_destination_project is not None) \
      and (ors_destination_project is not None) \
      and (user_destination_project == ors_destination_project)

    user_is_admin = 'administrator' in user_function_list
    user_is_ingestor = 'ingestor' in user_function_list

    # A client can only view a Data Acquisition Unit (ORS) if they are related to the same project
    # An administrator can view and edit all of them, as well as add one
    # An ingestor can view any of them
    self.checkDocumentPermissions(
      user,
      ingestion_item_dict['data_acquisition_unit'],
      same_project or user_is_admin or user_is_ingestor,
      user_is_admin,
      user_is_admin
    )

    # Same as above for a Data Supply (required for computing security roles on users)
    self.checkDocumentPermissions(
      user,
      ingestion_item_dict['data_supply'],
      same_project or user_is_admin or user_is_ingestor,
      user_is_admin,
      user_is_admin
    )

    # An administrator can view a Data Stream
    # An ingestor has all rights to a Data Stream (in order to append new log data to it)
    self.checkDocumentPermissions(user, ingestion_item_dict['data_stream'], user_is_admin or user_is_ingestor, user_is_ingestor, user_is_ingestor)

    # A client can only view a Data Analysis if they are related to the same project (required for KPI graphing)
    # An administrator can view all of them
    self.checkDocumentPermissions(user, ingestion_item_dict['data_analysis'], user_is_admin or same_project, False, False)

    # A client can only view a Data Array if they are related to the same project
    # An administrator can view all of them
    for data_array in ingestion_item_dict['data_array_list']:
      self.checkDocumentPermissions(user, data_array, user_is_admin or same_project, False, False)

  def checkModulePermissions(self, user):
    user_function_list = None
    for assignment in user.contentValues(portal_type='Assignment'):
      if assignment.getValidationState() == 'open':
        user_function_list = assignment.getFunctionList()

    user_is_client = 'user' in user_function_list
    user_is_admin = 'administrator' in user_function_list
    user_is_ingestor = 'ingestor' in user_function_list

    # Everyone can view the Data Product module (required for KPI graphing)
    # Everyone can also view the two data products used in the KPI calculation process (required for KPI graphing)
    self.checkDocumentPermissions(user, self.portal.data_product_module, True, False, False)
    ors_kpi = self.portal.portal_catalog.getResultValue(
      portal_type='Data Product',
      reference='ors_kpi',
      validation_state='validated'
    )
    self.checkDocumentPermissions(user, ors_kpi, True, False, False)
    ors_enb_log_data = self.portal.portal_catalog.getResultValue(
      portal_type='Data Product',
      reference='ors_enb_log_data',
      validation_state='validated'
    )
    self.checkDocumentPermissions(user, ors_enb_log_data, True, False, False)

    # Only ingestors can view the Data Operation Module, as well as the two Data Operations required
    # for ORS eNB log ingestion
    self.checkDocumentPermissions(user, self.portal.data_operation_module, user_is_ingestor, False, False)
    ingest_ors_enb_log_data = self.portal.portal_catalog.getResultValue(
      portal_type='Data Operation',
      reference='ingest_ors_enb_log_data',
      validation_state='validated'
    )
    self.checkDocumentPermissions(user, ingest_ors_enb_log_data, user_is_ingestor, False, False)
    calculate_ors_kpi = self.portal.portal_catalog.getResultValue(
      portal_type='Data Operation',
      reference='calculate_ors_kpi',
      validation_state='validated'
    )
    self.checkDocumentPermissions(user, calculate_ors_kpi, user_is_ingestor, False, False)

    # Everyone can view the Data Acquisition Unit and Data Supply modules
    # Only administrators can add items to them
    self.checkDocumentPermissions(user, self.portal.data_acquisition_unit_module, True, False, user_is_admin)
    self.checkDocumentPermissions(user, self.portal.data_supply_module, True, False, user_is_admin)

    # Only clients and administrator can view the Data Transformation module (required for KPI graphing)
    # Only they can also view the data transformation used to produce the KPIs (required for KPI graphing)
    self.checkDocumentPermissions(
      user,
      self.portal.data_transformation_module,
      user_is_client or user_is_admin,
      False,
      False
    )
    data_transformation = self.portal.portal_catalog.getResultValue(
      portal_type='Data Transformation',
      reference='ors_enb_log_data_transformation',
      validation_state='validated'
    )
    self.checkDocumentPermissions(user, data_transformation, user_is_client or user_is_admin, False, False)

    # Only ingestors and administrators can view the Data Ingestion and Data Stream modules
    # Only ingestors can add new Data Ingestions and Data Streams
    self.checkDocumentPermissions(
      user,
      self.portal.data_ingestion_module,
      user_is_ingestor or user_is_admin,
      False,
      user_is_ingestor
    )
    self.checkDocumentPermissions(
      user,
      self.portal.data_stream_module,
      user_is_ingestor or user_is_admin,
      False,
      user_is_ingestor
    )

    # Only administrators can view the Data Analysis module
    self.checkDocumentPermissions(user, self.portal.data_analysis_module, user_is_admin, False, False)

    # Only clients and administrators can view the Data Array module (required for KPI graphing)
    self.checkDocumentPermissions(user, self.portal.data_array_module, user_is_client or user_is_admin, False, False)

    # Only administrators have access to the Person and Project modules and can add items to them for client management purposes
    self.checkDocumentPermissions(user, self.portal.project_module, user_is_admin, False, user_is_admin)
    self.checkDocumentPermissions(user, self.portal.person_module, user_is_admin, False, user_is_admin)

  def test_01_createOrsDataSupply(self):
    '''
    Test the action which creates an ORS Data Supply from a Data Acquisition Unit.
    Check that the Data Supply is indeed created and validated.
    '''
    reference = 'test_%s' % generateRandomString()

    # Create and validate a Data Acquisition Unit
    data_acquisition_unit = self.portal.data_acquisition_unit_module.newContent(
      portal_type='Data Acquisition Unit',
      reference=reference
    )
    data_acquisition_unit.validate()
    self.tic()

    # Call the script which creates a related Data Supply
    created_data_supply = data_acquisition_unit.DataAcquisitionUnit_createOrsDataSupply(batch=1)
    self.tic()

    # Check that the Data Supply exists and is validated
    self.assertTrue(created_data_supply is not None)
    self.assertTrue(created_data_supply.getValidationState() == 'validated')

    # Call the script again to retrieve the same Data Supply
    retrieved_data_supply = data_acquisition_unit.DataAcquisitionUnit_createOrsDataSupply(batch=1)

    # Check that both Data Supplies are identical
    self.assertTrue(created_data_supply == retrieved_data_supply)

  def test_02_registerOrsClientProject(self):
    '''
    Test the action performed by Administrator users in the Project module to register a new client project.
    Check the successful case as well as all error cases.
    '''

    # Generate a random reference seed, call the script and retrieve the associated items
    # This first call should succeed
    reference_seed = generateRandomString()
    project_item_dict = self.registerOrsClientProject(reference_seed=reference_seed)

    # Check that both the project and the client user have been created
    self.assertTrue(project_item_dict['project'] is not None)
    self.assertTrue(project_item_dict['client_user'] is not None)

    # Call the script a second time with the same reference
    # This should not do anything as the project already exists
    repeated_project_item_dict = self.registerOrsClientProject(reference_seed=reference_seed)

    # Check that both the project and the client user are identical to the previous ones
    self.assertTrue(repeated_project_item_dict['project'] == project_item_dict['project'])
    self.assertTrue(repeated_project_item_dict['client_user'] == repeated_project_item_dict['client_user'])

    # Create a new reference seed for the project, but reuse the previous reference for the client user account
    new_project_reference_seed = generateRandomString()
    while new_project_reference_seed == reference_seed:
      new_project_reference_seed = generateRandomString()
    # Call the script a third time, keeping the same reference seed as before ONLY for the client user account
    # This should also error out as the client user account already exists
    new_project_item_dict = self.registerOrsClientProject(reference_seed=new_project_reference_seed, client_user_reference_seed=reference_seed)

    # Check that the new project is NOT created and that the client user is the same as previously
    self.assertTrue(new_project_item_dict['project'] is None)
    self.assertTrue(new_project_item_dict['client_user'] == project_item_dict['client_user'])

  def test_03_1_registerValidOrs(self):
    '''
    Test the script called during slave instantiation in SlapOS by an ORS to automatically register itself.
    Check all valid cases.
    '''

    # Call the script with an initial seed setup
    tag_hostname_seed = generateRandomString(length=3, only_digits=True)
    tag_comp_id_seed = generateRandomString(length=4, only_digits=True)
    tag_enb_id_seed = generateRandomString(length=5, hexadecimal=True)
    ors_item_dict = self.registerOrs(
      tag_hostname_seed=tag_hostname_seed,
      tag_comp_id_seed=tag_comp_id_seed,
      tag_enb_id_seed=tag_enb_id_seed
    )

    # Parse the JSON response and check that it indicates a success
    response_dict = json.loads(ors_item_dict['response'])
    self.assertTrue(response_dict['status'] == "ok")
    self.assertTrue(
      response_dict['message'] == "ORS with tag %s successfully registered." \
      % ors_item_dict['data_acquisition_unit'].getReference()
    )

    # Check that the Data Acquisition Unit and Data Supply have been created
    self.assertTrue(ors_item_dict['data_acquisition_unit'] is not None)
    self.assertTrue(ors_item_dict['data_supply'] is not None)

    # Call the script a second time with the same seeds
    # This should not do anything as the items already exist
    repeated_ors_item_dict = self.registerOrs(
      tag_hostname_seed=tag_hostname_seed,
      tag_comp_id_seed=tag_comp_id_seed,
      tag_enb_id_seed=tag_enb_id_seed
    )

    # Parse the JSON response and check the status and message
    response_dict = json.loads(repeated_ors_item_dict['response'])
    self.assertTrue(response_dict['status'] == "ok")
    self.assertTrue(
      response_dict['message'] == "ORS with tag %s already exists." \
      % ors_item_dict['data_acquisition_unit'].getReference()
    )

    # Now, link the original Data Supply to a client project
    project_a_item_dict = self.registerOrsClientProject()
    project_a_url = project_a_item_dict['project'].getRelativeUrl()
    ors_item_dict['data_supply'].setDestinationProject(project_a_url)

    # Generate a new valid enb_id seed
    new_tag_enb_id_seed = generateRandomString(length=5, hexadecimal=True)
    # Call the script to simulate an ORS re-registering with another eNB identifier
    new_enb_id_ors_item_dict = self.registerOrs(
      tag_hostname_seed=tag_hostname_seed,
      tag_comp_id_seed=tag_comp_id_seed,
      tag_enb_id_seed=new_tag_enb_id_seed
    )

    # Check that the Data Acquisition Unit and Data Supply have been created
    # And that the new Data Supply has automatically been linked to the project
    self.assertTrue(new_enb_id_ors_item_dict['data_acquisition_unit'] is not None)
    self.assertTrue(new_enb_id_ors_item_dict['data_supply'] is not None)
    self.assertTrue(new_enb_id_ors_item_dict['data_supply'].getDestinationProject() == project_a_url)

    # Now, link the above Data Supply to a second project
    project_b_item_dict = self.registerOrsClientProject()
    new_enb_id_ors_item_dict['data_supply'].setDestinationProject(project_b_item_dict['project'].getRelativeUrl())

    # Generate another valid enb_id seed
    another_tag_enb_id_seed = generateRandomString(length=5, hexadecimal=True)
    while another_tag_enb_id_seed == new_tag_enb_id_seed:
      another_tag_enb_id_seed = generateRandomString(length=5, hexadecimal=True)
    # Call the script to simulate the same ORS registering a third time with another eNB identifier
    another_enb_id_ors_item_dict = self.registerOrs(
      tag_hostname_seed=tag_hostname_seed,
      tag_comp_id_seed=tag_comp_id_seed,
      tag_enb_id_seed=another_tag_enb_id_seed
    )

    # Check that the Data Acquisition Unit and Data Supply have been created
    self.assertTrue(another_enb_id_ors_item_dict['data_acquisition_unit'] is not None)
    self.assertTrue(another_enb_id_ors_item_dict['data_supply'] is not None)
    # As the ORS has been linked to two different projects already,
    # it cannot be automatically decided to which project this version should be assigned to
    self.assertTrue(another_enb_id_ors_item_dict['data_supply'].getDestinationProject() is None)

    # Generate a new set of seeds
    new_tag_hostname_seed = generateRandomString(length=3, only_digits=True)
    new_tag_comp_id_seed = generateRandomString(length=4, only_digits=True)
    # Call the script with a tag that does not contain an eNB identifier
    no_enb_id_ors_item_dict = self.registerOrs(
      tag_hostname_seed=new_tag_hostname_seed,
      tag_comp_id_seed=new_tag_comp_id_seed,
      tag_enb_id_seed=''
    )

    # Parse the JSON response and check that it indicates a success
    response_dict = json.loads(no_enb_id_ors_item_dict['response'])
    self.assertTrue(response_dict['status'] == "ok")
    self.assertTrue(
      response_dict['message'] == "ORS with tag %s successfully registered." \
      % no_enb_id_ors_item_dict['data_acquisition_unit'].getReference()
    )

    # Check that the Data Acquisition Unit and Data Supply have been created
    self.assertTrue(no_enb_id_ors_item_dict['data_acquisition_unit'] is not None)
    self.assertTrue(no_enb_id_ors_item_dict['data_supply'] is not None)

  def test_03_2_registerInvalidOrs(self):
    '''
    Test the script called during slave instantiation in SlapOS by an ORS to automatically register itself.
    Check all invalid cases.
    '''

    tag_hostname_seed = generateRandomString(length=3, only_digits=True)
    tag_comp_id_seed = generateRandomString(length=4, only_digits=True)
    tag_enb_id_seed = generateRandomString(length=5, hexadecimal=True)

    # Generate a hostname seed that will cause the tag to have too many underscores
    too_long_tag_hostname_seed = 'invalid_hostname'
    # Call the script with this seed setup
    # This should error out as the tag has too many underscore-separated sections
    too_long_tag_ors_item_dict = self.registerOrs(
      tag_hostname_seed=too_long_tag_hostname_seed,
      tag_comp_id_seed=tag_comp_id_seed,
      tag_enb_id_seed=tag_enb_id_seed
    )

    # Parse the JSON response and check the error message
    response_dict = json.loads(too_long_tag_ors_item_dict['response'])
    self.assertTrue(response_dict['status'] == "error")
    self.assertTrue(
      response_dict['message'] == "Invalid ORS tag ors%s_COMP-%s_e0x%sTest found" \
      % (too_long_tag_hostname_seed, tag_comp_id_seed, tag_enb_id_seed)
    )

    # Check that the Data Acquisition Unit and Data Supply have NOT been created
    self.assertTrue(too_long_tag_ors_item_dict['data_acquisition_unit'] is None)
    self.assertTrue(too_long_tag_ors_item_dict['data_supply'] is None)

    # Now, call the script with only the first section in the tag: 'orsXXX'
    # This should error out as the tag has too few underscore-separated sections
    too_long_tag_ors_item_dict = self.registerOrs(
      tag_hostname_seed=tag_hostname_seed,
      tag_comp_id_seed='',
      tag_enb_id_seed=''
    )

    # Parse the JSON response and check the error message
    response_dict = json.loads(too_long_tag_ors_item_dict['response'])
    self.assertTrue(response_dict['status'] == "error")
    self.assertTrue(response_dict['message'] == "Invalid ORS tag ors%sTest found" % tag_hostname_seed)

    # Check that the Data Acquisition Unit and Data Supply have NOT been created
    self.assertTrue(too_long_tag_ors_item_dict['data_acquisition_unit'] is None)
    self.assertTrue(too_long_tag_ors_item_dict['data_supply'] is None)

  def test_04_1_ingestValidOrsLogDataFromFluentd(self, data_key="valid"):
    '''
    Test a simple valid ORS log ingestion: simulate a fluentd gateway forwarding valid ORS logs to the platform,
    and check that all items related to the ingestion are valid.
    '''
    # Register the ORS
    ors_item_dict = self.registerOrs()
    ors_tag = ors_item_dict['data_acquisition_unit'].getReference()

    # Get the correct data logs according to what is being tested
    test_ors_example_log = None
    if data_key == "valid":
      test_ors_example_log = self.test_ors_example_log_valid
    elif data_key == "invalid":
      test_ors_example_log = self.test_ors_example_log_invalid
    elif data_key == "empty":
      test_ors_example_log = self.test_ors_example_log_empty

    # Perform ingestion
    ingestion_item_dict = self.getOrsLogIngestionItems(test_ors_example_log, ors_tag)

    # In all cases, check that all items related to the ingestion exist
    self.assertEqual(NO_CONTENT, ingestion_item_dict['response'].getStatus())
    self.assertTrue(ingestion_item_dict['data_acquisition_unit'] is not None)
    self.assertTrue(ingestion_item_dict['data_supply'] is not None)
    self.assertTrue(ingestion_item_dict['data_stream'].getData() is not None)
    self.assertTrue(ingestion_item_dict['data_analysis'] is not None)
    self.assertTrue(all(data_array is not None for data_array in ingestion_item_dict['data_array_list']))

    # Check that the data arrays containing the KPI data have correctly been initialized
    self.assertTrue(ingestion_item_dict['data_acquisition_unit'].DataAcquisitionUnit_getERabDataArrayKey() is not None)
    self.assertTrue(ingestion_item_dict['data_acquisition_unit'].DataAcquisitionUnit_getEUtranDataArrayKey() is not None)

    e_rab_shape = (0,)
    e_rab_dtype = [
      ('vt', '<f8'),
      ('vInitialEPSBEstabSR_lo', '<f8'),
      ('vInitialEPSBEstabSR_hi', '<f8'),
      ('vAddedEPSBEstabSR_lo', '<f8'),
      ('vAddedEPSBEstabSR_hi', '<f8')
    ]
    e_utran_shape = (0,)
    e_utran_dtype = [('evt', '<f8'), ('qci', '<f8'), ('dl_lo', '<f8'), ('dl_hi', '<f8'), ('ul_lo', '<f8'), ('ul_hi', '<f8')]
    if data_key == "valid":
      e_rab_shape = (24,)
    elif data_key == "invalid":
      e_rab_shape = (30,)
    elif data_key == "empty":
      e_rab_shape = None
      e_rab_dtype = None
      e_utran_shape = None
      e_utran_dtype = None

    for data_array in ingestion_item_dict['data_array_list']:
      if 'e_rab' in data_array.getReference():
        self.assertTrue(data_array.getArrayDtype() == e_rab_dtype)
        self.assertTrue(data_array.getArrayShape() == e_rab_shape)
      elif 'e_utran' in data_array.getReference():
        self.assertTrue(data_array.getArrayDtype() == e_utran_dtype)
        self.assertTrue(data_array.getArrayShape() == e_utran_shape)

  def test_04_2_ingestInvalidOrsLogDataFromFluentd(self):
    '''
    Test an invalid ORS log ingestion: simulate a fluentd gateway forwarding invalid ORS logs to the platform.
    Check that all items are still valid, as only the invalid measurements are ignored.
    '''

    # Call the above test, but test with invalid data
    self.test_04_1_ingestValidOrsLogDataFromFluentd(data_key="invalid")

  def test_04_3_ingestEmptyOrsLogDataFromFluentd(self):
    '''
    Test an empty ORS log ingestion: simulate a fluentd gateway forwarding empty ORS logs to the platform.
    Check that all items are valid, but the data arrays remain uninitialized.
    '''

    # Call the above test, but test with empty data
    self.test_04_1_ingestValidOrsLogDataFromFluentd(data_key="empty")

  def test_04_4_ingestOrsLogDataWithoutPrefix(self):
    '''
    Simulate an entity trying to send data to the platform for ingestion
    without using the 'ors.' prefix added by fluentd.
    Check that the ingestion is refused.
    '''

    # No need to register an ORS here
    entity_tag = generateRandomString()

    # Call the script that parses the ingestion tag
    # Check that it raises the expected error
    self.assertRaises(
      ValueError,
      self.portal.IngestionPolicy_parseOrsFluentdTag,
      entity_tag
    )

  def test_05_wendelinTelecomSecurityModel(self):
    '''
    Test Wendelin Telecom's custom security model:
    check that different users have the correct permissions according to their function and project.
    '''

    # Setup two different projects with one linked ORS each
    project_a_item_dict = self.registerOrsClientProject()
    ors_a_item_dict = self.registerOrs()
    ors_a_tag = ors_a_item_dict['data_acquisition_unit'].getReference()

    project_b_item_dict = self.registerOrsClientProject()
    ors_b_item_dict = self.registerOrs()
    ors_b_tag = ors_b_item_dict['data_acquisition_unit'].getReference()

    # Register a third ORS that will not be linked to a project
    ors_n_item_dict = self.registerOrs()
    ors_n_tag = ors_n_item_dict['data_acquisition_unit'].getReference()

    # Perform ingestions for all three ORSs
    ingestion_a_item_dict = self.getOrsLogIngestionItems(
      self.test_ors_example_log_valid,
      ors_a_tag,
      stop_data_analysis=True
    )
    ingestion_b_item_dict = self.getOrsLogIngestionItems(
      self.test_ors_example_log_valid,
      ors_b_tag,
      stop_data_analysis=True
    )
    ingestion_n_item_dict = self.getOrsLogIngestionItems(
      self.test_ors_example_log_valid,
      ors_n_tag,
      stop_data_analysis=True
    )

    # Link the first two ORSs to their project AFTER the ingestion
    # To check that project propagation on ingestion objects works correctly
    ors_a_item_dict['data_supply'].setDestinationProject(project_a_item_dict['project'].getRelativeUrl())
    ors_b_item_dict['data_supply'].setDestinationProject(project_b_item_dict['project'].getRelativeUrl())

    # Create a client user not associated to a project
    client_user_n = self.createWendelinTelecomUser('test_user_%s' % generateRandomString(), None, 'user')

    # Create two administrator users: one associated to Project A and the second not associated to a project
    admin_user_a = self.createWendelinTelecomUser(
      'test_user_%s' % generateRandomString(),
      project_a_item_dict['project'].getRelativeUrl(),
      'administrator'
    )
    admin_user_n = self.createWendelinTelecomUser('test_administrator_%s' % generateRandomString(), None, 'administrator')

    # Check that the client of Project A only has access to Project A documents
    client_user_a = project_a_item_dict['client_user']
    self.checkModulePermissions(client_user_a)
    self.checkIngestionDocumentsPermissions(client_user_a, ingestion_a_item_dict)
    self.checkIngestionDocumentsPermissions(client_user_a, ingestion_b_item_dict)
    self.checkIngestionDocumentsPermissions(client_user_a, ingestion_n_item_dict)

    # Check that the client of project_B only has access to project_B documents
    client_user_b = project_b_item_dict['client_user']
    self.checkModulePermissions(client_user_b)
    self.checkIngestionDocumentsPermissions(client_user_b, ingestion_a_item_dict)
    self.checkIngestionDocumentsPermissions(client_user_b, ingestion_b_item_dict)
    self.checkIngestionDocumentsPermissions(client_user_b, ingestion_n_item_dict)

    # Check that the client without a project does not have access to any document
    self.checkModulePermissions(client_user_n)
    self.checkIngestionDocumentsPermissions(client_user_n, ingestion_a_item_dict)
    self.checkIngestionDocumentsPermissions(client_user_n, ingestion_b_item_dict)
    self.checkIngestionDocumentsPermissions(client_user_n, ingestion_n_item_dict)

    # Check that both administrators, whether assigned to a project, have access to all documents
    self.checkModulePermissions(admin_user_a)
    self.checkIngestionDocumentsPermissions(admin_user_a, ingestion_a_item_dict)
    self.checkIngestionDocumentsPermissions(admin_user_a, ingestion_b_item_dict)
    self.checkIngestionDocumentsPermissions(admin_user_a, ingestion_n_item_dict)

    self.checkModulePermissions(admin_user_a)
    self.checkIngestionDocumentsPermissions(admin_user_n, ingestion_a_item_dict)
    self.checkIngestionDocumentsPermissions(admin_user_n, ingestion_b_item_dict)
    self.checkIngestionDocumentsPermissions(admin_user_n, ingestion_n_item_dict)

    # Check that the ingestor user only has access to documents needed for ingestion
    self.checkModulePermissions(self.ingestor_user)
    self.checkIngestionDocumentsPermissions(self.ingestor_user, ingestion_a_item_dict)
    self.checkIngestionDocumentsPermissions(self.ingestor_user, ingestion_b_item_dict)
    self.checkIngestionDocumentsPermissions(self.ingestor_user, ingestion_n_item_dict)