Commit e36b68d4 authored by Rafael Monnerat's avatar Rafael Monnerat

slapos_slap_tool: Cache the XML to preserve performance

  Re-serialise the object from the dict after hit cache is slow enough
to cause problems ounder high load.

  This change preserve the previous behaviour reaching the final XML
from cache keeping performace on SlapTool and allowing new
implementation benefit from cache at same time.

  Fixup test to match xml with xml.
parent 0dbf8e50
......@@ -109,11 +109,14 @@ class SlapOSComputeNodeMixin(object):
def _fillComputeNodeInformationCache(self, user):
key = '%s_%s' % (self.getReference(), user)
try:
computer_dict = self._getCacheComputeNodeInformation(user)
self._getCachePlugin().set(key, DEFAULT_CACHE_SCOPE,
dict (
time=time.time(),
refresh_etag=self._calculateRefreshEtag(),
data=self._getCacheComputeNodeInformation(user),
data=computer_dict,
# Store the XML while SlapTool Still used
data_xml=self.getPortalObject().portal_slap._getSlapComputeNodeXMLFromDict(computer_dict)
),
cache_duration=self.getPortalObject().portal_caches\
.getRamCacheRoot().get('compute_node_information_cache_factory'\
......@@ -154,7 +157,6 @@ class SlapOSComputeNodeMixin(object):
user_document = _assertACI(portal.portal_catalog.unrestrictedGetResultValue(
reference=user, portal_type=['Person', 'Compute Node', 'Software Instance']))
user_type = user_document.getPortalType()
self.REQUEST.response.setHeader('Content-Type', 'text/xml; charset=utf-8')
if user_type in ('Compute Node', 'Person'):
if not self._isTestRun():
......@@ -191,7 +193,8 @@ class SlapOSComputeNodeMixin(object):
portal_type="Compute Partition",
checked_permission="View")
else:
compute_partition_list = self.getPortalObject().portal_catalog.unrestrictedSearchResults(
unrestrictedSearchResults = self.getPortalObject().portal_catalog.unrestrictedSearchResults
compute_partition_list = unrestrictedSearchResults(
parent_uid=self.getUid(),
validation_state="validated",
portal_type="Compute Partition")
......
......@@ -88,7 +88,9 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
self.commit()
first_etag = self.compute_node._calculateRefreshEtag()
first_body_fingerprint = hashData(
self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
self.portal_slap._getSlapComputeNodeXMLFromDict(
self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
)
)
self.assertEqual(200, response.status)
self.assertTrue('last-modified' not in response.headers)
......@@ -123,7 +125,9 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
self.assertTrue('last-modified' not in response.headers)
second_etag = self.compute_node._calculateRefreshEtag()
second_body_fingerprint = hashData(
self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
self.portal_slap._getSlapComputeNodeXMLFromDict(
self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
)
)
self.assertNotEqual(first_etag, second_etag)
# The indexation timestamp does not impact the response body
......@@ -156,7 +160,9 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
# Edition does not impact the etag
self.assertEqual(second_etag, self.compute_node._calculateRefreshEtag())
third_body_fingerprint = hashData(
self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
self.portal_slap._getSlapComputeNodeXMLFromDict(
self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
)
)
# The edition impacts the response body
self.assertNotEqual(first_body_fingerprint, third_body_fingerprint)
......@@ -195,7 +201,9 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
# The edition does not impact the response body yet, as the aggregate relation
# is not yet unindex
self.assertEqual(third_body_fingerprint, hashData(
self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
self.portal_slap._getSlapComputeNodeXMLFromDict(
self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
)
))
response = self.portal_slap.getFullComputerInformation(self.compute_node_id)
self.commit()
......@@ -217,7 +225,9 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
self.assertTrue('last-modified' not in response.headers)
fourth_etag = self.compute_node._calculateRefreshEtag()
fourth_body_fingerprint = hashData(
self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
self.portal_slap._getSlapComputeNodeXMLFromDict(
self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
)
)
self.assertNotEqual(third_etag, fourth_etag)
# The indexation timestamp does not impact the response body
......
......@@ -39,6 +39,8 @@ from Products.ERP5Type.Tool.BaseTool import BaseTool
from Products.ERP5Type import Permissions
from Products.ERP5Type.Cache import CachingMethod
from erp5.component.module.SlapOSCloud import _assertACI
from Products.ERP5Type.Cache import DEFAULT_CACHE_SCOPE
from lxml import etree
try:
from slapos.slap.slap import (
......@@ -169,60 +171,56 @@ class SlapTool(BaseTool):
Reuses slap library for easy marshalling.
"""
user = self.getPortalObject().portal_membership.getAuthenticatedMember().getUserName()
portal = self.getPortalObject()
user = portal.portal_membership.getAuthenticatedMember().getUserName()
if str(user) == computer_id:
compute_node = self.getPortalObject().portal_membership.getAuthenticatedMember().getUserValue()
compute_node = portal.portal_membership.getAuthenticatedMember().getUserValue()
compute_node.setAccessStatus(computer_id)
else:
# Don't use getDocument because we don't want use _assertACI here, but
# just call the API on computer.
compute_node = self.getPortalObject().portal_catalog.unrestrictedSearchResults(
compute_node = portal.portal_catalog.unrestrictedSearchResults(
portal_type='Compute Node', reference=computer_id,
validation_state="validated")[0].getObject()
refresh_etag = compute_node._calculateRefreshEtag()
computer_dict, etag = compute_node._getComputeNodeInformation(user, refresh_etag)
## Horrible code starts
# Convert computer node into SlapComputer
slap_compute_node = ComputeNode(computer_dict["_computer_id"])
slap_compute_node._computer_partition_list = []
slap_compute_node._software_release_list = []
for partition_dict in computer_dict["_computer_partition_list"]:
slap_compute_partition = SlapComputePartition(
partition_id=partition_dict["partition_id"],
computer_id=partition_dict['compute_node_id']
)
slap_compute_partition._requested_state = partition_dict["_requested_state"]
slap_compute_partition._need_modification = partition_dict["_need_modification"]
if partition_dict["_software_release_document"] is not None:
slap_compute_partition._access_status = partition_dict["_access_status"]
slap_compute_partition._parameter_dict = partition_dict["_parameter_dict"]
slap_compute_partition._connection_dict = partition_dict["_connection_dict"]
slap_compute_partition._filter_dict = partition_dict["_filter_dict"]
slap_compute_partition._instance_guid = partition_dict["_instance_guid"]
slap_compute_partition._software_release_document = SoftwareRelease(
software_release=partition_dict["_software_release_document"]["software_release"],
computer_guid=partition_dict["_software_release_document"]["computer_guid"])
# Keep compatibility with older clients that relies on marshalling.
# This code is an adaptation of SlapOSComputeNodeMixin._getComputeNodeInformation
# To comply with cache capability.
user_document = _assertACI(portal.portal_catalog.unrestrictedGetResultValue(
reference=user, portal_type=['Person', 'Compute Node', 'Software Instance']))
user_type = user_document.getPortalType()
if user_type in ('Compute Node', 'Person'):
if not compute_node._isTestRun():
cache_plugin = compute_node._getCachePlugin()
key = '%s_%s' % (compute_node.getReference(), user)
try:
entry = cache_plugin.get(key, DEFAULT_CACHE_SCOPE)
except KeyError:
entry = None
if entry is not None and isinstance(entry.getValue(), dict):
cached_dict = entry.getValue()
cached_etag = cached_dict.get('refresh_etag', None)
if (refresh_etag != cached_etag):
# Do not recalculate the compute_node information
# if nothing changed
compute_node._activateFillComputeNodeInformationCache(user)
etag = cached_etag
body = cached_dict['data_xml']
else:
compute_node._activateFillComputeNodeInformationCache(user)
self.REQUEST.response.setStatus(503)
return self.REQUEST.response
else:
slap_compute_partition._software_release_document = None
slap_compute_node._computer_partition_list.append(
slap_compute_partition
)
for software_release_dict in computer_dict['_software_release_list']:
slap_software_release = SoftwareRelease(
software_release=software_release_dict["software_release"],
computer_guid=software_release_dict['computer_guid'])
slap_software_release._requested_state = software_release_dict['_requested_state']
slap_software_release._known_state = software_release_dict['_known_state']
slap_compute_node._software_release_list.append(slap_software_release)
body = dumps(slap_compute_node)
## Horrible code ends
computer_dict, etag = compute_node._getComputeNodeInformation(user, refresh_etag)
body = self._getSlapComputeNodeXMLFromDict(computer_dict)
else:
computer_dict, etag = compute_node._getComputeNodeInformation(user, refresh_etag)
body = self._getSlapComputeNodeXMLFromDict(computer_dict)
self.REQUEST.response.setHeader('Content-Type', 'text/xml; charset=utf-8')
if self.REQUEST.response.getStatus() == 200:
# Keep in cache server for 7 days
self.REQUEST.response.setHeader('Cache-Control',
......@@ -734,7 +732,6 @@ class SlapTool(BaseTool):
####################################################
# Internal methods
####################################################
def _validateXML(self, to_be_validated, xsd_model):
"""Will validate the xml file"""
#We parse the XSD model
......@@ -1012,6 +1009,44 @@ class SlapTool(BaseTool):
# Internals methods
####################################################
def _getSlapComputeNodeXMLFromDict(self, computer_dict):
slap_compute_node = ComputeNode(computer_dict["_computer_id"])
slap_compute_node._computer_partition_list = []
slap_compute_node._software_release_list = []
for partition_dict in computer_dict["_computer_partition_list"]:
slap_compute_partition = SlapComputePartition(
partition_id=partition_dict["partition_id"],
computer_id=partition_dict['compute_node_id']
)
slap_compute_partition._requested_state = partition_dict["_requested_state"]
slap_compute_partition._need_modification = partition_dict["_need_modification"]
if partition_dict["_software_release_document"] is not None:
slap_compute_partition._access_status = partition_dict["_access_status"]
slap_compute_partition._parameter_dict = partition_dict["_parameter_dict"]
slap_compute_partition._connection_dict = partition_dict["_connection_dict"]
slap_compute_partition._filter_dict = partition_dict["_filter_dict"]
slap_compute_partition._instance_guid = partition_dict["_instance_guid"]
slap_compute_partition._software_release_document = SoftwareRelease(
software_release=partition_dict["_software_release_document"]["software_release"],
computer_guid=partition_dict["_software_release_document"]["computer_guid"])
else:
slap_compute_partition._software_release_document = None
slap_compute_node._computer_partition_list.append(
slap_compute_partition
)
for software_release_dict in computer_dict['_software_release_list']:
slap_software_release = SoftwareRelease(
software_release=software_release_dict["software_release"],
computer_guid=software_release_dict['computer_guid'])
slap_software_release._requested_state = software_release_dict['_requested_state']
slap_software_release._known_state = software_release_dict['_known_state']
slap_compute_node._software_release_list.append(slap_software_release)
return dumps(slap_compute_node)
def _getSoftwareInstanceForComputePartition(self, compute_node_id,
compute_partition_id, slave_reference=None):
compute_partition_document = self.getPortalObject().portal_catalog.getComputePartitionObject(
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment