Skip to content

S
slapos.core
Commit 693c8c5c authored by Rafael Monnerat's avatar Rafael Monnerat
Browse files
Options

easily monitor compute node which has modified file 

See merge request !503
parents 69f8d5be 183bb94a
Pipeline #29330 failed with stage
    in 0 seconds
    Hide whitespace changes
    Inline Side-by-side
    Showing with 640 additions and 3 deletions
    master/bt5/slapos_abyss/ActionTemplateItem/portal_types/Data%20Array/declare_as_default_data.xml 0 → 100644
    View file @ 693c8c5c
    <?xml version="1.0"?>
    <ZopeData>
    <record id="1" aka="AAAAAAAAAAE=">
    <pickle>
    <global name="ActionInformation" module="Products.CMFCore.ActionInformation"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>action</string> </key>
    <value>
    <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
    </value>
    </item>
    <item>
    <key> <string>categories</string> </key>
    <value>
    <tuple>
    <string>action_type/object_jio_action</string>
    </tuple>
    </value>
    </item>
    <item>
    <key> <string>category</string> </key>
    <value> <string>object_jio_action</string> </value>
    </item>
    <item>
    <key> <string>condition</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>description</string> </key>
    <value>
    <none/>
    </value>
    </item>
    <item>
    <key> <string>icon</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>id</string> </key>
    <value> <string>declare_as_default_data</string> </value>
    </item>
    <item>
    <key> <string>permissions</string> </key>
    <value>
    <tuple>
    <string>Modify portal content</string>
    </tuple>
    </value>
    </item>
    <item>
    <key> <string>portal_type</string> </key>
    <value> <string>Action Information</string> </value>
    </item>
    <item>
    <key> <string>priority</string> </key>
    <value> <float>1.0</float> </value>
    </item>
    <item>
    <key> <string>title</string> </key>
    <value> <string>Declare As Default Data</string> </value>
    </item>
    <item>
    <key> <string>visible</string> </key>
    <value> <int>1</int> </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    <record id="2" aka="AAAAAAAAAAI=">
    <pickle>
    <global name="Expression" module="Products.CMFCore.Expression"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>text</string> </key>
    <value> <string>string:${object_url}/DataArray_viewDeclareAsDefaultDataDialog</string> </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    </ZopeData>
    master/bt5/slapos_abyss/PathTemplateItem/portal_callables/DataArray_processDataArray.py
    View file @ 693c8c5c
    ......@@ -88,6 +88,7 @@ if next_database_image:
    new_data_array = context.data_array_module.newContent(portal_type='Data Array')
    new_data_array.setArray(diff_array)
    new_data_array.edit(
    reference= '%s-%s-%s' % (context.getReference(), next_database_image.getReference(), new_data_array.getId()),
    title='diff of %s and %s' %(context.getTitle(), next_database_image.getTitle()),
    predecessor_value_list=[context, next_database_image],
    publication_section=["file_system_image/node_image", "file_system_image/process_state/converted"],
    ......
    master/bt5/slapos_abyss/SkinTemplateItem/portal_skins/slapos_abyss/ComputeNode_hasModifiedFile.py
    View file @ 693c8c5c
    ......@@ -2,6 +2,7 @@ from Products.ZSQLCatalog.SQLCatalog import AndQuery, Query
    query = AndQuery(
    Query(portal_type = ["Data Array"]),
    Query(validation_state='validated'),
    Query(causality_uid=context.getUid()),
    AndQuery(Query(publication_section_relative_url = 'publication_section/file_system_image/diff_end'),
    Query(publication_section_relative_url = 'publication_section/file_system_image/process_state/processed')))
    ......
    master/bt5/slapos_abyss/SkinTemplateItem/portal_skins/slapos_abyss/DataArray_declareAsDefaultData.py 0 → 100644
    View file @ 693c8c5c
    from Products.ZSQLCatalog.SQLCatalog import SimpleQuery, NegatedQuery
    current_publication_section_list = context.getPublicationSectionList()
    portal = context.getPortalObject()
    if 'file_system_image/first_access' not in current_publication_section_list:
    current_publication_section_list.append('file_system_image/first_access')
    context.setPublicationSectionList(current_publication_section_list)
    if portal.portal_workflow.isTransitionPossible(context, 'validate'):
    context.validate(comment='declare as default data')
    for previous_array in portal.portal_catalog(
    portal_type='Data Array',
    query=NegatedQuery(SimpleQuery(uid=context.getUid())),
    publication_section_relative_url ='publication_section/file_system_image/first_access',
    causality_uid = context.getCausalityUid(),
    validation_state='validated'
    ):
    previous_array.invalidate(comment="%s is declared as default data" % context.getRelativeUrl())
    if batch:
    return
    context.Base_redirect('view',keep_items={'portal_status_message': context.Base_translateString("Declared as default data")})
    master/bt5/slapos_abyss/SkinTemplateItem/portal_skins/slapos_abyss/DataArray_declareAsDefaultData.xml 0 → 100644
    View file @ 693c8c5c
    <?xml version="1.0"?>
    <ZopeData>
    <record id="1" aka="AAAAAAAAAAE=">
    <pickle>
    <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>_bind_names</string> </key>
    <value>
    <object>
    <klass>
    <global name="_reconstructor" module="copy_reg"/>
    </klass>
    <tuple>
    <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
    <global name="object" module="__builtin__"/>
    <none/>
    </tuple>
    <state>
    <dictionary>
    <item>
    <key> <string>_asgns</string> </key>
    <value>
    <dictionary>
    <item>
    <key> <string>name_container</string> </key>
    <value> <string>container</string> </value>
    </item>
    <item>
    <key> <string>name_context</string> </key>
    <value> <string>context</string> </value>
    </item>
    <item>
    <key> <string>name_m_self</string> </key>
    <value> <string>script</string> </value>
    </item>
    <item>
    <key> <string>name_subpath</string> </key>
    <value> <string>traverse_subpath</string> </value>
    </item>
    </dictionary>
    </value>
    </item>
    </dictionary>
    </state>
    </object>
    </value>
    </item>
    <item>
    <key> <string>_params</string> </key>
    <value> <string>batch = 0, **kw</string> </value>
    </item>
    <item>
    <key> <string>id</string> </key>
    <value> <string>DataArray_declareAsDefaultData</string> </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    </ZopeData>
    master/bt5/slapos_abyss/SkinTemplateItem/portal_skins/slapos_abyss/DataArray_viewDeclareAsDefaultDataDialog.xml 0 → 100644
    View file @ 693c8c5c
    <?xml version="1.0"?>
    <ZopeData>
    <record id="1" aka="AAAAAAAAAAE=">
    <pickle>
    <global name="ERP5 Form" module="erp5.portal_type"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>_objects</string> </key>
    <value>
    <tuple/>
    </value>
    </item>
    <item>
    <key> <string>action</string> </key>
    <value> <string>DataArray_declareAsDefaultData</string> </value>
    </item>
    <item>
    <key> <string>action_title</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>description</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>edit_order</string> </key>
    <value>
    <list/>
    </value>
    </item>
    <item>
    <key> <string>encoding</string> </key>
    <value> <string>UTF-8</string> </value>
    </item>
    <item>
    <key> <string>enctype</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>group_list</string> </key>
    <value>
    <list>
    <string>left</string>
    <string>right</string>
    <string>center</string>
    <string>bottom</string>
    <string>hidden</string>
    </list>
    </value>
    </item>
    <item>
    <key> <string>groups</string> </key>
    <value>
    <dictionary>
    <item>
    <key> <string>bottom</string> </key>
    <value>
    <list/>
    </value>
    </item>
    <item>
    <key> <string>center</string> </key>
    <value>
    <list/>
    </value>
    </item>
    <item>
    <key> <string>hidden</string> </key>
    <value>
    <list/>
    </value>
    </item>
    <item>
    <key> <string>left</string> </key>
    <value>
    <list>
    <string>your_warning_text</string>
    </list>
    </value>
    </item>
    <item>
    <key> <string>right</string> </key>
    <value>
    <list/>
    </value>
    </item>
    </dictionary>
    </value>
    </item>
    <item>
    <key> <string>id</string> </key>
    <value> <string>DataArray_viewDeclareAsDefaultDataDialog</string> </value>
    </item>
    <item>
    <key> <string>method</string> </key>
    <value> <string>POST</string> </value>
    </item>
    <item>
    <key> <string>name</string> </key>
    <value> <string>DataArray_declareAsDefaultDataDialog</string> </value>
    </item>
    <item>
    <key> <string>pt</string> </key>
    <value> <string>form_dialog</string> </value>
    </item>
    <item>
    <key> <string>row_length</string> </key>
    <value> <int>4</int> </value>
    </item>
    <item>
    <key> <string>stored_encoding</string> </key>
    <value> <string>UTF-8</string> </value>
    </item>
    <item>
    <key> <string>title</string> </key>
    <value> <string>Declare As Default Data</string> </value>
    </item>
    <item>
    <key> <string>unicode_mode</string> </key>
    <value> <int>0</int> </value>
    </item>
    <item>
    <key> <string>update_action</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>update_action_title</string> </key>
    <value> <string></string> </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    </ZopeData>
    master/bt5/slapos_abyss/SkinTemplateItem/portal_skins/slapos_abyss/DataArray_viewDeclareAsDefaultDataDialog/your_warning_text.xml 0 → 100644
    View file @ 693c8c5c
    <?xml version="1.0"?>
    <ZopeData>
    <record id="1" aka="AAAAAAAAAAE=">
    <pickle>
    <global name="EditorField" module="Products.ERP5Form.EditorField"/>
    </pickle>
    <pickle>
    <dictionary>
    <item>
    <key> <string>id</string> </key>
    <value> <string>your_warning_text</string> </value>
    </item>
    <item>
    <key> <string>message_values</string> </key>
    <value>
    <dictionary>
    <item>
    <key> <string>external_validator_failed</string> </key>
    <value> <string>The input failed the external validator.</string> </value>
    </item>
    <item>
    <key> <string>line_too_long</string> </key>
    <value> <string>A line was too long.</string> </value>
    </item>
    <item>
    <key> <string>required_not_found</string> </key>
    <value> <string>Input is required but no input given.</string> </value>
    </item>
    <item>
    <key> <string>too_long</string> </key>
    <value> <string>You entered too many characters.</string> </value>
    </item>
    <item>
    <key> <string>too_many_lines</string> </key>
    <value> <string>You entered too many lines.</string> </value>
    </item>
    </dictionary>
    </value>
    </item>
    <item>
    <key> <string>overrides</string> </key>
    <value>
    <dictionary>
    <item>
    <key> <string>alternate_name</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>css_class</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>default</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>description</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>editable</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>enabled</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>external_validator</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>extra</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>height</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>hidden</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>max_length</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>max_linelength</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>max_lines</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>required</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>text_editor</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>title</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>unicode</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>whitespace_preserve</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>width</string> </key>
    <value> <string></string> </value>
    </item>
    </dictionary>
    </value>
    </item>
    <item>
    <key> <string>tales</string> </key>
    <value>
    <dictionary>
    <item>
    <key> <string>alternate_name</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>css_class</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>default</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>description</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>editable</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>enabled</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>external_validator</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>extra</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>height</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>hidden</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>max_length</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>max_linelength</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>max_lines</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>required</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>text_editor</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>title</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>unicode</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>whitespace_preserve</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>width</string> </key>
    <value> <string></string> </value>
    </item>
    </dictionary>
    </value>
    </item>
    <item>
    <key> <string>values</string> </key>
    <value>
    <dictionary>
    <item>
    <key> <string>alternate_name</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>css_class</string> </key>
    <value> <string>hidden_label</string> </value>
    </item>
    <item>
    <key> <string>default</string> </key>
    <value> <string encoding="cdata"><![CDATA[
    <b>WARNING:</b> This will make current data as default, previous data will be invalidated
    ]]></string> </value>
    </item>
    <item>
    <key> <string>description</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>editable</string> </key>
    <value> <int>0</int> </value>
    </item>
    <item>
    <key> <string>enabled</string> </key>
    <value> <int>1</int> </value>
    </item>
    <item>
    <key> <string>external_validator</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>extra</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>height</string> </key>
    <value> <int>5</int> </value>
    </item>
    <item>
    <key> <string>hidden</string> </key>
    <value> <int>0</int> </value>
    </item>
    <item>
    <key> <string>max_length</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>max_linelength</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>max_lines</string> </key>
    <value> <string></string> </value>
    </item>
    <item>
    <key> <string>required</string> </key>
    <value> <int>0</int> </value>
    </item>
    <item>
    <key> <string>text_editor</string> </key>
    <value> <string>text_area</string> </value>
    </item>
    <item>
    <key> <string>title</string> </key>
    <value> <string>Warning</string> </value>
    </item>
    <item>
    <key> <string>unicode</string> </key>
    <value> <int>0</int> </value>
    </item>
    <item>
    <key> <string>whitespace_preserve</string> </key>
    <value> <int>0</int> </value>
    </item>
    <item>
    <key> <string>width</string> </key>
    <value> <int>40</int> </value>
    </item>
    </dictionary>
    </value>
    </item>
    </dictionary>
    </pickle>
    </record>
    </ZopeData>
    master/bt5/slapos_abyss/TestTemplateItem/portal_components/test.erp5.testSlapOSAbyss.py
    View file @ 693c8c5c
    ......@@ -654,6 +654,7 @@ class testSlapOSAbyss(SlapOSTestCaseMixin):
    self.tic()
    self.portal.portal_alarms.slapos_process_data_array.activeSense()
    self.tic()
    default_array = self._getRelatedDataArrayList(self.portal.compute_node_module['node_debian10'])[-1]
    self.portal.portal_alarms.wendelin_handle_analysis.activeSense()
    self.tic()
    self.portal.portal_alarms.slapos_process_data_array.activeSense()
    ......@@ -661,8 +662,12 @@ class testSlapOSAbyss(SlapOSTestCaseMixin):
    self.assertTrue(self.portal.compute_node_module['node_debian10'].ComputeNode_hasModifiedFile() is None)
    self.portal.portal_alarms.wendelin_handle_analysis.activeSense()
    self.tic()
    modified_array = self._getRelatedDataArrayList(self.portal.compute_node_module['node_debian10'])[-1]
    self.portal.portal_alarms.slapos_process_data_array.activeSense()
    self.tic()
    last_diff_array = self._getRelatedDataArrayList(self.portal.compute_node_module['node_debian10'])[-1]
    self.assertTrue(modified_array.getReference() in last_diff_array.getReference())
    self.assertTrue(default_array.getReference() in last_diff_array.getReference())
    self.portal.portal_alarms.slapos_process_data_array.activeSense()
    self.tic()
    self.portal.portal_alarms.slapos_process_data_array.activeSense()
    ......@@ -670,7 +675,30 @@ class testSlapOSAbyss(SlapOSTestCaseMixin):
    # no more database to compare
    self.portal.portal_alarms.slapos_process_data_array.activeSense()
    self.tic()
    self.assertTrue(self.portal.compute_node_module['node_debian10'].ComputeNode_hasModifiedFile() is not None)
    data_array = self.portal.compute_node_module['node_debian10'].ComputeNode_hasModifiedFile()
    self.assertTrue(data_array is not None)
    data_array.invalidate()
    self.tic()
    self.assertTrue(self.portal.compute_node_module['node_debian10'].ComputeNode_hasModifiedFile() is None)
    modified_array.DataArray_declareAsDefaultData(batch=1)
    self.tic()
    self.assertEqual(modified_array.getValidationState(), 'validated')
    self.assertEqual(default_array.getValidationState(), 'invalidated')
    # still the same
    modified_array.DataArray_declareAsDefaultData(batch=1)
    self.tic()
    self.assertEqual(modified_array.getValidationState(), 'validated')
    self.assertEqual(default_array.getValidationState(), 'invalidated')
    # ingest again, this time as default array is changed, it should has no more difference
    self._ingestData(request_dict)
    self.tic()
    self.portal.portal_alarms.wendelin_handle_analysis.activeSense()
    self.tic()
    self.portal.portal_alarms.slapos_process_data_array.activeSense()
    self.tic()
    self.assertTrue('file_system_image/diff_end/identical' in self._getRelatedDataArrayList(self.portal.compute_node_module['node_debian10'])[-1].getPublicationSectionList())
    self.assertTrue(self.portal.compute_node_module['node_debian10'].ComputeNode_hasModifiedFile() is None)
    def test_data_processing_check_value_in_data_array(self):
    ......
    master/bt5/slapos_abyss/bt/template_action_path_list
    View file @ 693c8c5c
    ......@@ -3,4 +3,5 @@ Compute Node | create_data_transformation
    Compute Node | jump_to_data_array
    Compute Node | jump_to_data_stream
    Compute Node | operating_system_file_info_view
    Data Array | declare_as_default_data
    Data Array | mapped_content
    \ No newline at end of file
    master/bt5/slapos_crm/SkinTemplateItem/portal_skins/slapos_crm_monitoring/ComputeNode_checkState.py
    View file @ 693c8c5c
    ......@@ -16,6 +16,7 @@ ticket_title = node_ticket_title
    description = ""
    last_contact = "No Contact Information"
    issue_document_reference = ""
    notification_message_reference = 'slapos-crm-compute_node_check_state.notification'
    now = DateTime()
    ......@@ -36,8 +37,10 @@ else:
    data_array = context.ComputeNode_hasModifiedFile()
    if data_array:
    should_notify = True
    notification_message_reference = "slapos-crm-compute_node_check_modified_file.notification"
    ticket_title = "[MONITORING] Compute Node %s has modified file" % reference
    description = "The Compute Node %s (%s) has modified file: %s" % (compute_node_title, reference, data_array.getRelativeUrl())
    issue_document_reference = data_array.Reference()
    description = "The Compute Node %s (%s) has modified file: %s" % (compute_node_title, reference, issue_document_reference)
    if not should_notify:
    # Since server is contacting, check for stalled processes
    ......@@ -107,7 +110,8 @@ if should_notify:
    else:
    mapping_dict = {'compute_node_title':context.getTitle(),
    'compute_node_id':reference,
    'last_contact':last_contact}
    'last_contact':last_contact,
    'issue_document_reference': issue_document_reference}
    message = notification_message.asText(
    substitution_method_parameter_dict={'mapping_dict': mapping_dict})
    ......
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment
    GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7