slapos_crm: Check if Slaved instance is left over

   If a software instance with 1 or more slave instances allocated on it is destroyed the instances aren't reallocated, so let's create a ticket to high light the problem.

master/bt5/slapos_crm/SkinTemplateItem/portal_skins/slapos_crm_monitoring/SoftwareInstance_getReportedErrorDict.py

@@ -38,6 +38,17 @@ if compute_node.getPortalType() == "Compute Node" and \
   error_dict['ticket_description'] = error_dict['message']
   return error_dict
 
+if context.getPortalType() == 'Slave Instance' and compute_node.getPortalType() == "Compute Node":
+  software_instance = compute_partition.getAggregateRelated(portal_type='Software Instance')
+  if software_instance is None:
+    raise ValueError(compute_partition.getRelativeUrl())
+    # Slave instance is allocated but the software instance was already destroyed
+    error_dict['notification_message_reference'] = 'slapos-crm-instance-tree-slave-on-destroyed-instance.notification'
+    error_dict = updateErrorDictWithError(error_dict)
+    error_dict['message'] = "%s is allocated on a destroyed software instance (already removed)." % context.getTitle()
+    error_dict['ticket_description'] = error_dict['message']
+    return error_dict
+
 # Skip to check if monitor disabled on the compute node.
 # Remote node has no state.
 if (compute_node.getPortalType() == "Compute Node") and (compute_node.getMonitorScope() != "enabled"):

master/bt5/slapos_crm/TestTemplateItem/portal_components/test.erp5.testSlapOSCRMMonitoringAlarm.py

@@ -912,7 +912,8 @@ class TestSlapOSCrmSoftwareInstance_checkInstanceTreeMonitoringState(TestSlapOSC
   @simulate('Project_isSupportRequestCreationClosed', '', 'return 0')
   def test_SoftwareInstance_checkInstanceTreeMonitoringState_script_slaveInstance(self):
     with PinnedDateTime(self, DateTime() - 1.1):
-      _, _, _, _, _, instance_tree = self.bootstrapAllocableInstanceTree(allocation_state='allocated', shared=True)
+      _, _, _, _, _, instance_tree = self.bootstrapAllocableInstanceTree(
+        node='instance', allocation_state='allocated', shared=True)
       software_instance = instance_tree.getSuccessorValue()
       software_instance.setErrorStatus("foo")
 
@@ -944,6 +945,45 @@ class TestSlapOSCrmSoftwareInstance_checkInstanceTreeMonitoringState(TestSlapOSC
     self.assertEqual(event.getSimulationState(), "delivered")
     self.assertEqual(event.getPortalType(), "Web Message")
 
+
+  @simulate('Project_isSupportRequestCreationClosed', '', 'return 0')
+  def test_SoftwareInstance_checkInstanceTreeMonitoringState_script_slaveWithoutSoftwareInstance(self):
+    with PinnedDateTime(self, DateTime() - 1.1):
+      # without instance='node', the slave is allocated w/o software instance
+      # which emulates post unallocation state after the destruction of 
+      # the software instance.
+      _, _, _, _, _, instance_tree = self.bootstrapAllocableInstanceTree(
+        allocation_state='allocated', shared=True)
+      software_instance = instance_tree.getSuccessorValue()
+    self.tic()
+
+    error_dict = software_instance.SoftwareInstance_getReportedErrorDict()
+    software_instance.SoftwareInstance_checkInstanceTreeMonitoringState()
+    self.tic()
+
+    ticket = self._getGeneratedSupportRequest(instance_tree.getUid())
+    self.assertEqual(error_dict['should_notify'], True)
+
+    ticket_title =  "Instance Tree %s is failing." % (instance_tree.getTitle())
+    self.assertEqual(error_dict['ticket_title'], ticket_title)
+    ticket = self._getGeneratedSupportRequest(instance_tree.getUid())
+
+    self.assertNotEqual(ticket, None)
+    self.assertEqual(ticket.getTitle(), error_dict['ticket_title'])
+    event_list = ticket.getFollowUpRelatedValueList()
+    self.assertEqual(len(event_list), 1)
+    event = event_list[0]
+
+    self.assertIn('is allocated on a destroyed software instance',
+                  event.getTextContent())
+    self.assertEqual(event.getFollowUp(), ticket.getRelativeUrl())
+    self.assertEqual(event.getSourceProject(), instance_tree.getFollowUp())
+    self.assertEqual(ticket.getSourceProject(), instance_tree.getFollowUp())
+    self.assertEqual(ticket.getCausality(), instance_tree.getRelativeUrl())
+    self.assertEqual(ticket.getSimulationState(), "submitted")
+    self.assertEqual(event.getSimulationState(), "delivered")
+    self.assertEqual(event.getPortalType(), "Web Message")
+
   @simulate('Project_isSupportRequestCreationClosed', '', 'return 0')
   def test_SoftwareInstance_checkInstanceTreeMonitoringState_script_instanceOnRemoteNode(self):
     with PinnedDateTime(self, DateTime() - 1.1):