Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
W
wendelin
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Léo-Paul Géneau
wendelin
Commits
bc99f630
Commit
bc99f630
authored
Mar 08, 2021
by
Roque
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
erp5_wendelin_data_lake_ingestion: fix state-consistency tests
parent
08c288da
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
10 additions
and
6 deletions
+10
-6
bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.py
...Item/portal_components/test.erp5.testDataLakeIngestion.py
+9
-5
bt5/erp5_wendelin_data_lake_ingestion/WorkflowTemplateItem/portal_workflow/data_set_interaction_workflow/scripts/DataSet_invalidateDataStreamList.py
...tion_workflow/scripts/DataSet_invalidateDataStreamList.py
+1
-1
No files found.
bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.py
View file @
bc99f630
...
@@ -227,7 +227,7 @@ class TestDataIngestion(SecurityTestCase):
...
@@ -227,7 +227,7 @@ class TestDataIngestion(SecurityTestCase):
self
.
tic
()
self
.
tic
()
# check data relation between Data Set and Data Streams work
# check data relation between Data Set and Data Streams work
self
.
assertSameSet
(
data_stream_list
,
data_set
.
DataSet_getDataStreamList
()
)
self
.
assertSameSet
(
data_stream_list
,
[
x
.
getObject
()
for
x
in
data_set
.
DataSet_getDataStreamList
()]
)
# check data set and all Data Streams states
# check data set and all Data Streams states
self
.
assertEqual
(
'validated'
,
data_set
.
getValidationState
())
self
.
assertEqual
(
'validated'
,
data_set
.
getValidationState
())
...
@@ -340,15 +340,18 @@ class TestDataIngestion(SecurityTestCase):
...
@@ -340,15 +340,18 @@ class TestDataIngestion(SecurityTestCase):
"""
"""
# ingest a couple of files
# ingest a couple of files
reference
=
self
.
getRandomReference
()
reference
=
self
.
getRandomReference
()
self
.
ingest
(
"some-data-1"
,
reference
,
self
.
CSV
,
self
.
SINGLE_INGESTION_END
)
data_set_reference
=
"test-relation-dataset"
self
.
ingest
(
"some-data-1"
,
reference
,
self
.
CSV
,
self
.
SINGLE_INGESTION_END
,
randomize_ingestion_reference
=
False
,
data_set_reference
=
data_set_reference
)
time
.
sleep
(
1
)
time
.
sleep
(
1
)
self
.
tic
()
self
.
tic
()
reference
+=
"-2"
reference
+=
"-2"
self
.
ingest
(
"some-data-2"
,
reference
,
self
.
CSV
,
self
.
SINGLE_INGESTION_END
)
self
.
ingest
(
"some-data-2"
,
reference
,
self
.
CSV
,
self
.
SINGLE_INGESTION_END
,
randomize_ingestion_reference
=
False
,
data_set_reference
=
data_set_reference
)
time
.
sleep
(
1
)
time
.
sleep
(
1
)
self
.
tic
()
self
.
tic
()
# get corresponding Data Streams by searching via Data Ingestion Lines of the Data Set
# get corresponding Data Streams by searching via Data Ingestion Lines of the Data Set
data_set
=
self
.
portal
.
data_set_module
.
get
(
self
.
REF_DATASET
)
data_set
=
self
.
portal
.
data_set_module
.
get
(
data_set_reference
)
data_ingestion_line_list
=
self
.
portal
.
portal_catalog
(
data_ingestion_line_list
=
self
.
portal
.
portal_catalog
(
portal_type
=
'Data Ingestion Line'
,
portal_type
=
'Data Ingestion Line'
,
aggregate_uid
=
data_set
.
getUid
())
aggregate_uid
=
data_set
.
getUid
())
...
@@ -356,9 +359,10 @@ class TestDataIngestion(SecurityTestCase):
...
@@ -356,9 +359,10 @@ class TestDataIngestion(SecurityTestCase):
data_stream_list
=
self
.
portal
.
portal_catalog
(
data_stream_list
=
self
.
portal
.
portal_catalog
(
portal_type
=
'Data Stream'
,
portal_type
=
'Data Stream'
,
aggregate__related__uid
=
data_ingestion_uid_list
,
aggregate__related__uid
=
data_ingestion_uid_list
,
validation_state
=
[
'validated'
,
'published'
],
select_list
=
[
'reference'
,
'relative_url'
,
'versioning.size'
,
'versioning.version'
])
select_list
=
[
'reference'
,
'relative_url'
,
'versioning.size'
,
'versioning.version'
])
data_stream_list
=
[
x
.
getObject
()
for
x
in
data_stream_list
]
data_stream_list
=
[
x
.
getObject
()
for
x
in
data_stream_list
]
# assert that the list from the search is the same as DataSet_getDataStreamList
# assert that the list from the search is the same as DataSet_getDataStreamList
self
.
assertSameSet
(
data_stream_list
,
data_set
.
DataSet_getDataStreamList
()
)
self
.
assertSameSet
(
data_stream_list
,
[
x
.
getObject
()
for
x
in
data_set
.
DataSet_getDataStreamList
()]
)
# XXX: new test which simulates download / upload of Data Set and increase DS version
# XXX: new test which simulates download / upload of Data Set and increase DS version
\ No newline at end of file
bt5/erp5_wendelin_data_lake_ingestion/WorkflowTemplateItem/portal_workflow/data_set_interaction_workflow/scripts/DataSet_invalidateDataStreamList.py
View file @
bc99f630
"""
"""
Publish
all Data Streams for a Data Set.
Invalidate
all Data Streams for a Data Set.
"""
"""
data_set
=
state_change
[
'object'
]
data_set
=
state_change
[
'object'
]
for
data_stream
in
data_set
.
DataSet_getDataStreamList
():
for
data_stream
in
data_set
.
DataSet_getDataStreamList
():
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment