Commit a49edabc authored by Roque Porcchetto's avatar Roque Porcchetto

erp5_wendelin_telecom_ingestion: metadata for tsv files

parent cfce9d1e
......@@ -70,15 +70,17 @@ def getMNEReportJSON(file_name, raw):
def getRawData(file_name):
raw = None
raw_functions = [mne.io.read_raw_fif, mne.io.read_raw_edf, mne.io.read_events_eeglab, mne.io.read_annotations_eeglab, mne.io.read_raw_eeglab,
mne.io.read_raw_artemis123, mne.io.read_raw_bti, mne.io.read_raw_cnt, mne.io.read_raw_ctf, mne.io.read_raw_brainvision,
mne.io.read_raw_kit, mne.read_epochs_kit, mne.io.read_raw_nicolet, mne.io.read_raw_eeglab, mne.io.read_raw_brainvision,
mne.io.read_raw_egi, mne.io.read_info, mne.io.read_epochs_eeglab]
for function in raw_functions:
try:
raw = mne.io.read_raw_fif(file_name, preload=False, verbose=None)
except:
pass
if raw is None:
try:
raw = mne.io.read_raw_edf(file_name, preload=False, verbose=None)
except:
pass
raw = function(file_name)
print("Function %s could read file %s " % (str(function), file_name))
break
except Exception as e:
print("Exception reading raw data with function %s: %s" % (str(function), str(e)))
if raw is None: raise StandardError("the file does not contain raw data.")
return raw
......@@ -117,7 +119,7 @@ def processTextData(file_name, data_array, data_descriptor):
except Exception as e:
log("Error handling Data Descriptor content: " + str(e))
def processCsvData(file_name, data_array, data_descriptor):
def processCsvData(file_name, data_array, data_descriptor, delimiter=","):
def gen_csv_chunks(reader, chunksize=CHUNK_SIZE_CSV):
chunk = []
for index, line in enumerate(reader):
......@@ -149,7 +151,7 @@ def processCsvData(file_name, data_array, data_descriptor):
try:
with open(file_name, 'r') as csv_file:
reader = csv.reader(csv_file)
reader = csv.reader(csv_file, delimiter=delimiter)
csv_file.seek(0)
for i, chunk in enumerate(gen_csv_chunks(reader)):
if i==0:
......@@ -174,6 +176,21 @@ def processCsvData(file_name, data_array, data_descriptor):
except Exception as e:
log("Error handling csv Data Descriptor content: " + str(e))
def processTsvData(file_name, data_array, data_descriptor):
processCsvData(file_name, data_array, data_descriptor, delimiter="\t")
def processNiiData(file_name, data_array, data_descriptor):
data = {}
data["File content sample: "] = "nifti files data processing not available yet."
json_data = json.dumps(data)
data_descriptor.setTextContent(json_data)
def processGZData(file_name, data_array, data_descriptor):
if data_array.getReference().endswith(".nii/gz"):
processNiiData(file_name, data_array, data_descriptor)
else:
raise KeyError("gz")
def processRawData(data_stream, data_array, data_descriptor, reference_extension):
import time
start = time.time()
......@@ -187,8 +204,12 @@ def processRawData(data_stream, data_array, data_descriptor, reference_extension
return "Error while processing raw data - saving file: " + str(e)
options = {"fif" : processFifData,
"nii" : processNiiData,
"mgz" : processNiiData,
"txt" : processTextData,
"csv" : processCsvData,
"tsv" : processTsvData,
"gz" : processGZData,
"default" : processTextData,
}
try:
......
......@@ -52,11 +52,9 @@
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple>
<string>W: 75, 2: No exception type(s) specified (bare-except)</string>
<string>W: 80, 4: No exception type(s) specified (bare-except)</string>
<string>W: 98, 8: Unused variable \'times\' (unused-variable)</string>
<string>W:135, 6: No exception type(s) specified (bare-except)</string>
<string>W:171, 8: Unreachable code (unreachable)</string>
<string>W:100, 8: Unused variable \'times\' (unused-variable)</string>
<string>W:137, 6: No exception type(s) specified (bare-except)</string>
<string>W:173, 8: Unreachable code (unreachable)</string>
</tuple>
</value>
</item>
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment