Commit 300a725f authored by Roque Porchetto's avatar Roque Porchetto

erp5_wendelin_telecom_ingestion: unit test for nifty nii files

parent 68c616d9
...@@ -20,6 +20,8 @@ class TestDataIngestion(SecurityTestCase): ...@@ -20,6 +20,8 @@ class TestDataIngestion(SecurityTestCase):
TXT = "/txt" TXT = "/txt"
CSV = "/csv" CSV = "/csv"
TSV = "/tsv" TSV = "/tsv"
GZ = "/gz"
NII = "/nii"
RANDOM = "/" + ''.join([random.choice(string.ascii_letters + string.digits) for _ in xrange(3)]) RANDOM = "/" + ''.join([random.choice(string.ascii_letters + string.digits) for _ in xrange(3)])
CHUNK_SIZE_TXT = 50000 CHUNK_SIZE_TXT = 50000
CHUNK_SIZE_CSV = 25 CHUNK_SIZE_CSV = 25
...@@ -184,6 +186,44 @@ class TestDataIngestion(SecurityTestCase): ...@@ -184,6 +186,44 @@ class TestDataIngestion(SecurityTestCase):
if os.path.exists(file_name): if os.path.exists(file_name):
os.remove(file_name) os.remove(file_name)
def perform_nii_test(self, gz):
import nibabel as nib
import numpy as np
reference = self.getRandomReference()
url = 'data_stream_module/mne_sample_for_test_nii'
if gz: url += "_gz"
sample_data_stream = self.context.restrictedTraverse(url)
data_chunk = sample_data_stream.getData()
ingestion_reference = self.ingest(data_chunk, reference, self.GZ if gz else self.NII)
file_name = "file_name.nii"
if gz: file_name += ".gz"
with open(file_name, 'w') as nii_file:
nii_file.write(data_chunk)
img = nib.load(file_name)
data = {}
for key in img.header:
try:
if type(img.header[key]) is np.ndarray:
content = img.header[key].tolist()
try:
if np.isnan(img.header[key]): content = "nan"
except:
pass
else:
content = img.header[key]
if content == 0: content = "0"
json.dumps(content)
if content != "":
data[key] = content
except Exception as e:
pass # ignore non serializable info
json_data = json.dumps(data)
if os.path.exists(file_name):
os.remove(file_name)
self.checkDataObjects(ingestion_reference, data_chunk, None, json_data)
def test_full_data_ingestion(self): def test_full_data_ingestion(self):
reference = self.getRandomReference() reference = self.getRandomReference()
data_chunk, nparray, json_data = self.generateRawDataBytesAndArray() data_chunk, nparray, json_data = self.generateRawDataBytesAndArray()
...@@ -217,6 +257,12 @@ class TestDataIngestion(SecurityTestCase): ...@@ -217,6 +257,12 @@ class TestDataIngestion(SecurityTestCase):
self.checkDataObjects(ingestion_reference, data_chunk, None, json_data) self.checkDataObjects(ingestion_reference, data_chunk, None, json_data)
def test_nii_data_ingestion(self):
self.perform_nii_test(gz=False)
def test_nii_gz_data_ingestion(self):
self.perform_nii_test(gz=True)
def test_data_ingestion_splitted_file(self): def test_data_ingestion_splitted_file(self):
reference = self.getRandomReference() reference = self.getRandomReference()
ingestion_policy = self.getIngestionPolicy(reference, self.INGESTION_SCRIPT) ingestion_policy = self.getIngestionPolicy(reference, self.INGESTION_SCRIPT)
......
...@@ -46,9 +46,14 @@ ...@@ -46,9 +46,14 @@
<key> <string>text_content_warning_message</string> </key> <key> <string>text_content_warning_message</string> </key>
<value> <value>
<tuple> <tuple>
<string>W:166, 34: Unused variable \'i\' (unused-variable)</string> <string>W:168, 34: Unused variable \'i\' (unused-variable)</string>
<string>W:166, 76: Unused variable \'j\' (unused-variable)</string> <string>W:168, 76: Unused variable \'j\' (unused-variable)</string>
<string>W:230, 4: Unused variable \'ingestion_id\' (unused-variable)</string> <string>W:191, 4: Redefining name \'np\' from outer scope (line 9) (redefined-outer-name)</string>
<string>W:191, 4: Reimport \'numpy\' (imported line 9) (reimported)</string>
<string>W:207, 11: Using type() instead of isinstance() for a typecheck. (unidiomatic-typecheck)</string>
<string>W:211, 10: No exception type(s) specified (bare-except)</string>
<string>W:219, 26: Unused variable \'e\' (unused-variable)</string>
<string>W:276, 4: Unused variable \'ingestion_id\' (unused-variable)</string>
</tuple> </tuple>
</value> </value>
</item> </item>
......
...@@ -11,4 +11,6 @@ portal_categories/function/** ...@@ -11,4 +11,6 @@ portal_categories/function/**
portal_categories/use/big_data/ingestion/batch_ingestion portal_categories/use/big_data/ingestion/batch_ingestion
data_stream_module/embulk_download_script data_stream_module/embulk_download_script
data_stream_module/embulk_download_script_win data_stream_module/embulk_download_script_win
data_stream_module/mne_sample_for_test data_stream_module/mne_sample_for_test
\ No newline at end of file data_stream_module/mne_sample_for_test_nii
data_stream_module/mne_sample_for_test_nii_gz
\ No newline at end of file
...@@ -11,4 +11,6 @@ portal_categories/function/** ...@@ -11,4 +11,6 @@ portal_categories/function/**
portal_categories/use/big_data/ingestion/batch_ingestion portal_categories/use/big_data/ingestion/batch_ingestion
data_stream_module/embulk_download_script data_stream_module/embulk_download_script
data_stream_module/embulk_download_script_win data_stream_module/embulk_download_script_win
data_stream_module/mne_sample_for_test data_stream_module/mne_sample_for_test
\ No newline at end of file data_stream_module/mne_sample_for_test_nii
data_stream_module/mne_sample_for_test_nii_gz
\ No newline at end of file
...@@ -6,6 +6,8 @@ data_product_module/fif_descriptor ...@@ -6,6 +6,8 @@ data_product_module/fif_descriptor
data_stream_module/embulk_download_script data_stream_module/embulk_download_script
data_stream_module/embulk_download_script_win data_stream_module/embulk_download_script_win
data_stream_module/mne_sample_for_test data_stream_module/mne_sample_for_test
data_stream_module/mne_sample_for_test_nii
data_stream_module/mne_sample_for_test_nii_gz
data_supply_module/** data_supply_module/**
data_transformation_module/** data_transformation_module/**
person_module/** person_module/**
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment