Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
W
wendelin
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Kirill Smelkov
wendelin
Commits
b042daae
Commit
b042daae
authored
Feb 17, 2021
by
Roque
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
erp5_wendelin_data_lake_ingestion: new tests
parent
20ec3314
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
56 additions
and
3 deletions
+56
-3
bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.py
...Item/portal_components/test.erp5.testDataLakeIngestion.py
+54
-1
bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.xml
...tem/portal_components/test.erp5.testDataLakeIngestion.xml
+2
-2
No files found.
bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.py
View file @
b042daae
...
...
@@ -22,6 +22,7 @@ class TestDataIngestion(SecurityTestCase):
REF_PREFIX
=
"fake-supplier"
+
REFERENCE_SEPARATOR
REF_SUPPLIER_PREFIX
=
"fake-supplier"
+
REFERENCE_SEPARATOR
INVALID
=
"_invalid"
REF_DATASET
=
"fake-dataset"
def
getTitle
(
self
):
return
"DataIngestionTest"
...
...
@@ -38,7 +39,7 @@ class TestDataIngestion(SecurityTestCase):
def
getIngestionReference
(
self
,
reference
,
extension
,
randomize_ingestion_reference
=
False
,
data_set_reference
=
False
):
if
not
data_set_reference
:
data_set_reference
=
"fake-dataset"
data_set_reference
=
self
.
REF_DATASET
if
not
randomize_ingestion_reference
:
# return hard coded which results in one Data Set and multiple Data Streams (in context of test)
return
self
.
REF_PREFIX
+
data_set_reference
+
self
.
REFERENCE_SEPARATOR
+
reference
+
extension
...
...
@@ -281,4 +282,56 @@ class TestDataIngestion(SecurityTestCase):
self
.
assertEqual
(
first_file_stream
.
getValidationState
(),
'published'
)
self
.
assertEqual
(
second_file_stream
.
getValidationState
(),
'published'
)
def
test_06_IngestSameElementTwice
(
self
):
"""
Try to ingest same element twice while previous is being processed
"""
reference
=
self
.
getRandomReference
()
self
.
ingest
(
"some-data"
,
reference
,
self
.
CSV
,
self
.
SINGLE_INGESTION_END
)
time
.
sleep
(
1
)
self
.
tic
()
#ingest with same reference while previous was not processed should fail
self
.
assertRaises
(
Exception
,
self
.
ingest
,
"some-data"
,
reference
,
self
.
CSV
,
self
.
SINGLE_INGESTION_END
)
def
test_07_IngestionReplacesOld
(
self
):
"""
Try to ingest same element twice while previous is being processed
"""
reference
=
self
.
getRandomReference
()
self
.
ingest
(
"some-data"
,
reference
,
self
.
CSV
,
self
.
SINGLE_INGESTION_END
)
time
.
sleep
(
1
)
self
.
tic
()
# call explicitly alarm to process and validate the ingestion
self
.
portal
.
portal_alarms
.
wendelin_handle_analysis
.
Alarm_handleAnalysis
()
self
.
tic
()
ingestion_reference
=
self
.
REF_DATASET
+
self
.
REFERENCE_SEPARATOR
+
reference
+
self
.
CSV
data_ingestion
=
self
.
getDataIngestion
(
ingestion_reference
)
data_stream
=
self
.
getDataStream
(
ingestion_reference
)
self
.
assertEqual
(
data_ingestion
.
getSimulationState
(),
"stopped"
)
self
.
assertEqual
(
data_stream
.
getValidationState
(),
"validated"
)
# the new ingestion should replace the old one
self
.
ingest
(
"some-data"
,
reference
,
self
.
CSV
,
self
.
SINGLE_INGESTION_END
)
time
.
sleep
(
1
)
self
.
tic
()
self
.
assertEqual
(
data_stream
.
getValidationState
(),
"invalidated"
)
self
.
assertTrue
(
data_ingestion
.
getReference
().
endswith
(
self
.
INVALID
))
def
test_08_checkAlarms
(
self
):
"""
Check all data analysis and data transformations are well configured
"""
# call two times the alarm (1. create analysis 2. run analysis)
self
.
portal
.
portal_alarms
.
wendelin_handle_analysis
.
Alarm_handleAnalysis
()
self
.
tic
()
self
.
portal
.
portal_alarms
.
wendelin_handle_analysis
.
Alarm_handleAnalysis
()
self
.
tic
()
# XXX: new test which simulates download / upload of Data Set and increase DS version
\ No newline at end of file
bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.xml
View file @
b042daae
...
...
@@ -46,8 +46,8 @@
<key>
<string>
text_content_warning_message
</string>
</key>
<value>
<tuple>
<string>
W:10
5
, 34: Unused variable \'i\' (unused-variable)
</string>
<string>
W:10
5
, 76: Unused variable \'j\' (unused-variable)
</string>
<string>
W:10
6
, 34: Unused variable \'i\' (unused-variable)
</string>
<string>
W:10
6
, 76: Unused variable \'j\' (unused-variable)
</string>
</tuple>
</value>
</item>
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment