slapos_slap_tool: Cache the XML to preserve performance

  Re-serialise the object from the dict after hit cache is slow enough
to cause problems ounder high load.

  This change preserve the previous behaviour reaching the final XML
from cache keeping performace on SlapTool and allowing new
implementation benefit from cache at same time.

  Fixup test to match xml with xml.

master/bt5/slapos_cloud/MixinTemplateItem/portal_components/mixin.erp5.SlapOSComputeNodeMixin.py

@@ -109,11 +109,14 @@ class SlapOSComputeNodeMixin(object):
   def _fillComputeNodeInformationCache(self, user):
     key = '%s_%s' % (self.getReference(), user)
     try:
+      computer_dict = self._getCacheComputeNodeInformation(user)
       self._getCachePlugin().set(key, DEFAULT_CACHE_SCOPE,
         dict (
           time=time.time(),
           refresh_etag=self._calculateRefreshEtag(),
-          data=self._getCacheComputeNodeInformation(user),
+          data=computer_dict,
+          # Store the XML while SlapTool Still used
+          data_xml=self.getPortalObject().portal_slap._getSlapComputeNodeXMLFromDict(computer_dict)
         ),
         cache_duration=self.getPortalObject().portal_caches\
             .getRamCacheRoot().get('compute_node_information_cache_factory'\
@@ -154,7 +157,6 @@ class SlapOSComputeNodeMixin(object):
     user_document = _assertACI(portal.portal_catalog.unrestrictedGetResultValue(
       reference=user, portal_type=['Person', 'Compute Node', 'Software Instance']))
     user_type = user_document.getPortalType()
-    self.REQUEST.response.setHeader('Content-Type', 'text/xml; charset=utf-8')
 
     if user_type in ('Compute Node', 'Person'):
       if not self._isTestRun():
@@ -191,7 +193,8 @@ class SlapOSComputeNodeMixin(object):
         portal_type="Compute Partition",
         checked_permission="View")
     else:
-      compute_partition_list = self.getPortalObject().portal_catalog.unrestrictedSearchResults(
+      unrestrictedSearchResults = self.getPortalObject().portal_catalog.unrestrictedSearchResults
+      compute_partition_list = unrestrictedSearchResults(
                     parent_uid=self.getUid(),
                     validation_state="validated",
                     portal_type="Compute Partition")

master/bt5/slapos_slap_tool/TestTemplateItem/portal_components/test.erp5.testSlapOSSlapTool.py

@@ -88,7 +88,9 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
     self.commit()
     first_etag = self.compute_node._calculateRefreshEtag()
     first_body_fingerprint = hashData(
-      self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
+      self.portal_slap._getSlapComputeNodeXMLFromDict(
+        self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
+      )
     )
     self.assertEqual(200, response.status)
     self.assertTrue('last-modified' not in response.headers)
@@ -123,7 +125,9 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
     self.assertTrue('last-modified' not in response.headers)
     second_etag = self.compute_node._calculateRefreshEtag()
     second_body_fingerprint = hashData(
-      self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
+      self.portal_slap._getSlapComputeNodeXMLFromDict(
+        self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
+      )
     )
     self.assertNotEqual(first_etag, second_etag)
     # The indexation timestamp does not impact the response body
@@ -156,7 +160,9 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
     # Edition does not impact the etag
     self.assertEqual(second_etag, self.compute_node._calculateRefreshEtag())
     third_body_fingerprint = hashData(
-      self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
+      self.portal_slap._getSlapComputeNodeXMLFromDict(
+        self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
+      )
     )
     # The edition impacts the response body
     self.assertNotEqual(first_body_fingerprint, third_body_fingerprint)
@@ -195,7 +201,9 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
     # The edition does not impact the response body yet, as the aggregate relation
     # is not yet unindex
     self.assertEqual(third_body_fingerprint, hashData(
-      self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
+      self.portal_slap._getSlapComputeNodeXMLFromDict(
+        self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
+      )
     ))
     response = self.portal_slap.getFullComputerInformation(self.compute_node_id)
     self.commit()
@@ -217,7 +225,9 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
     self.assertTrue('last-modified' not in response.headers)
     fourth_etag = self.compute_node._calculateRefreshEtag()
     fourth_body_fingerprint = hashData(
-      self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
+      self.portal_slap._getSlapComputeNodeXMLFromDict(
+        self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
+      )
     )
     self.assertNotEqual(third_etag, fourth_etag)
     # The indexation timestamp does not impact the response body

master/bt5/slapos_slap_tool/ToolComponentTemplateItem/portal_components/tool.erp5.SlapTool.py

@@ -39,6 +39,8 @@ from Products.ERP5Type.Tool.BaseTool import BaseTool
 from Products.ERP5Type import Permissions
 from Products.ERP5Type.Cache import CachingMethod
 from erp5.component.module.SlapOSCloud import _assertACI
+from Products.ERP5Type.Cache import DEFAULT_CACHE_SCOPE
+
 from lxml import etree
 try:
   from slapos.slap.slap import (
@@ -169,60 +171,56 @@ class SlapTool(BaseTool):
 
     Reuses slap library for easy marshalling.
     """
-    user = self.getPortalObject().portal_membership.getAuthenticatedMember().getUserName()
+    portal = self.getPortalObject()
+    user = portal.portal_membership.getAuthenticatedMember().getUserName()
     if str(user) == computer_id:
-      compute_node = self.getPortalObject().portal_membership.getAuthenticatedMember().getUserValue()
+      compute_node = portal.portal_membership.getAuthenticatedMember().getUserValue()
       compute_node.setAccessStatus(computer_id)
     else:
       # Don't use getDocument because we don't want use _assertACI here, but
       # just call the API on computer.
-      compute_node = self.getPortalObject().portal_catalog.unrestrictedSearchResults(
+      compute_node = portal.portal_catalog.unrestrictedSearchResults(
         portal_type='Compute Node', reference=computer_id,
         validation_state="validated")[0].getObject()
 
     refresh_etag = compute_node._calculateRefreshEtag()
-    computer_dict, etag = compute_node._getComputeNodeInformation(user, refresh_etag)
-
-    ## Horrible code starts
-    # Convert computer node into SlapComputer
-    slap_compute_node = ComputeNode(computer_dict["_computer_id"])
-    slap_compute_node._computer_partition_list = []
-    slap_compute_node._software_release_list = []
 
-    for partition_dict in computer_dict["_computer_partition_list"]:
-      slap_compute_partition = SlapComputePartition(
-          partition_id=partition_dict["partition_id"],
-          computer_id=partition_dict['compute_node_id']
-        )
-      slap_compute_partition._requested_state = partition_dict["_requested_state"]
-      slap_compute_partition._need_modification = partition_dict["_need_modification"]
-      if partition_dict["_software_release_document"] is not None:
-        slap_compute_partition._access_status = partition_dict["_access_status"]
-        slap_compute_partition._parameter_dict = partition_dict["_parameter_dict"]
-        slap_compute_partition._connection_dict = partition_dict["_connection_dict"]
-        slap_compute_partition._filter_dict = partition_dict["_filter_dict"]
-        slap_compute_partition._instance_guid = partition_dict["_instance_guid"]
-        slap_compute_partition._software_release_document = SoftwareRelease(
-            software_release=partition_dict["_software_release_document"]["software_release"],
-            computer_guid=partition_dict["_software_release_document"]["computer_guid"])
+    # Keep compatibility with older clients that relies on marshalling.
+    # This code is an adaptation of SlapOSComputeNodeMixin._getComputeNodeInformation
+    # To comply with cache capability.
+    user_document = _assertACI(portal.portal_catalog.unrestrictedGetResultValue(
+      reference=user, portal_type=['Person', 'Compute Node', 'Software Instance']))
+    user_type = user_document.getPortalType()
+    if user_type in ('Compute Node', 'Person'):
+      if not compute_node._isTestRun():
+        cache_plugin = compute_node._getCachePlugin()
+        key = '%s_%s' % (compute_node.getReference(), user)
+        try:
+          entry = cache_plugin.get(key, DEFAULT_CACHE_SCOPE)
+        except KeyError:
+          entry = None
+
+        if entry is not None and isinstance(entry.getValue(), dict):
+          cached_dict = entry.getValue()
+          cached_etag = cached_dict.get('refresh_etag', None)
+          if (refresh_etag != cached_etag):
+            # Do not recalculate the compute_node information
+            # if nothing changed
+            compute_node._activateFillComputeNodeInformationCache(user)
+          etag = cached_etag
+          body = cached_dict['data_xml']
+        else:
+          compute_node._activateFillComputeNodeInformationCache(user)
+          self.REQUEST.response.setStatus(503)
+          return self.REQUEST.response
       else:
-        slap_compute_partition._software_release_document = None
-      
-      slap_compute_node._computer_partition_list.append(
-        slap_compute_partition
-      )
-
-    for software_release_dict in computer_dict['_software_release_list']:
-      slap_software_release = SoftwareRelease(
-        software_release=software_release_dict["software_release"],
-        computer_guid=software_release_dict['computer_guid'])
-      slap_software_release._requested_state = software_release_dict['_requested_state']
-      slap_software_release._known_state = software_release_dict['_known_state']
-      slap_compute_node._software_release_list.append(slap_software_release)
-
-    body = dumps(slap_compute_node)
-    ## Horrible code ends
+        computer_dict, etag = compute_node._getComputeNodeInformation(user, refresh_etag)
+        body = self._getSlapComputeNodeXMLFromDict(computer_dict)
+    else:
+      computer_dict, etag = compute_node._getComputeNodeInformation(user, refresh_etag)
+      body = self._getSlapComputeNodeXMLFromDict(computer_dict)
 
+    self.REQUEST.response.setHeader('Content-Type', 'text/xml; charset=utf-8')
     if self.REQUEST.response.getStatus() == 200:
       # Keep in cache server for 7 days
       self.REQUEST.response.setHeader('Cache-Control',
@@ -734,7 +732,6 @@ class SlapTool(BaseTool):
   ####################################################
   # Internal methods
   ####################################################
-
   def _validateXML(self, to_be_validated, xsd_model):
     """Will validate the xml file"""
     #We parse the XSD model
@@ -1012,6 +1009,44 @@ class SlapTool(BaseTool):
   # Internals methods
   ####################################################
 
+  def _getSlapComputeNodeXMLFromDict(self, computer_dict):
+    slap_compute_node = ComputeNode(computer_dict["_computer_id"])
+    slap_compute_node._computer_partition_list = []
+    slap_compute_node._software_release_list = []
+
+    for partition_dict in computer_dict["_computer_partition_list"]:
+      slap_compute_partition = SlapComputePartition(
+          partition_id=partition_dict["partition_id"],
+          computer_id=partition_dict['compute_node_id']
+        )
+      slap_compute_partition._requested_state = partition_dict["_requested_state"]
+      slap_compute_partition._need_modification = partition_dict["_need_modification"]
+      if partition_dict["_software_release_document"] is not None:
+        slap_compute_partition._access_status = partition_dict["_access_status"]
+        slap_compute_partition._parameter_dict = partition_dict["_parameter_dict"]
+        slap_compute_partition._connection_dict = partition_dict["_connection_dict"]
+        slap_compute_partition._filter_dict = partition_dict["_filter_dict"]
+        slap_compute_partition._instance_guid = partition_dict["_instance_guid"]
+        slap_compute_partition._software_release_document = SoftwareRelease(
+            software_release=partition_dict["_software_release_document"]["software_release"],
+            computer_guid=partition_dict["_software_release_document"]["computer_guid"])
+      else:
+        slap_compute_partition._software_release_document = None
+      
+      slap_compute_node._computer_partition_list.append(
+        slap_compute_partition
+      )
+
+    for software_release_dict in computer_dict['_software_release_list']:
+      slap_software_release = SoftwareRelease(
+        software_release=software_release_dict["software_release"],
+        computer_guid=software_release_dict['computer_guid'])
+      slap_software_release._requested_state = software_release_dict['_requested_state']
+      slap_software_release._known_state = software_release_dict['_known_state']
+      slap_compute_node._software_release_list.append(slap_software_release)
+
+    return dumps(slap_compute_node)
+
   def _getSoftwareInstanceForComputePartition(self, compute_node_id,
       compute_partition_id, slave_reference=None):
     compute_partition_document = self.getPortalObject().portal_catalog.getComputePartitionObject(