Commit 9a274e9e authored by Rafael Monnerat's avatar Rafael Monnerat

reduce computer xml calculation

See merge request nexedi/slapos.core!305
parents 01c2661d bb71f2d6
...@@ -43,7 +43,7 @@ CREATE TABLE `catalog` ( ...@@ -43,7 +43,7 @@ CREATE TABLE `catalog` (
`has_cell_content` bool, `has_cell_content` bool,
`creation_date` datetime, `creation_date` datetime,
`modification_date` datetime, `modification_date` datetime,
`indexation_timestamp` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, `indexation_timestamp` TIMESTAMP(6) DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
PRIMARY KEY (`uid`), PRIMARY KEY (`uid`),
KEY `security_uid` (`security_uid`), KEY `security_uid` (`security_uid`),
KEY `group_security_uid` (`group_security_uid`), KEY `group_security_uid` (`group_security_uid`),
...@@ -65,5 +65,6 @@ CREATE TABLE `catalog` ( ...@@ -65,5 +65,6 @@ CREATE TABLE `catalog` (
KEY `causality_state_portal_type` (`causality_state`, `portal_type`), KEY `causality_state_portal_type` (`causality_state`, `portal_type`),
KEY `invoice_state` (`invoice_state`), KEY `invoice_state` (`invoice_state`),
KEY `payment_state` (`payment_state`), KEY `payment_state` (`payment_state`),
KEY `event_state` (`event_state`) KEY `event_state` (`event_state`),
KEY `indexation_timestamp` (`indexation_timestamp`)
) ENGINE=InnoDB; ) ENGINE=InnoDB;
...@@ -43,3 +43,15 @@ def Item_activateFillComputerInformationCache(state_change): ...@@ -43,3 +43,15 @@ def Item_activateFillComputerInformationCache(state_change):
computer_reference, computer_reference) computer_reference, computer_reference)
finally: finally:
setSecurityManager(sm) setSecurityManager(sm)
@UnrestrictedMethod
def reindexPartition(item):
partition = item.getAggregateValue(portal_type='Computer Partition')
if partition is not None:
partition.reindexObject()
def Instance_reindexComputerPartition(state_change):
item = state_change['object']
reindexPartition(item)
...@@ -14,8 +14,15 @@ import xml.dom.ext.reader.Sax ...@@ -14,8 +14,15 @@ import xml.dom.ext.reader.Sax
import xml.dom.ext import xml.dom.ext
import StringIO import StringIO
import difflib import difflib
import hashlib
from binascii import hexlify
from OFS.Traversable import NotFound from OFS.Traversable import NotFound
def hashData(data):
return hexlify(hashlib.sha1(data).digest())
class Simulator: class Simulator:
def __init__(self, outfile, method): def __init__(self, outfile, method):
self.outfile = outfile self.outfile = outfile
...@@ -63,6 +70,166 @@ class TestSlapOSSlapToolMixin(SlapOSTestCaseMixin): ...@@ -63,6 +70,166 @@ class TestSlapOSSlapToolMixin(SlapOSTestCaseMixin):
self.unpinDateTime() self.unpinDateTime()
self._cleaupREQUEST() self._cleaupREQUEST()
class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
def test_activate_getFullComputerInformation_first_access(self):
self._makeComplexComputer(with_slave=True)
self.portal.REQUEST['disable_isTestRun'] = True
self.tic()
self.login(self.computer_user_id)
# First access.
# Cache has been filled by interaction workflow
# (luckily, it seems the cache is filled after everything is indexed)
response = self.portal_slap.getFullComputerInformation(self.computer_id)
self.commit()
first_etag = self.portal_slap._calculateRefreshEtag()
first_body_fingerprint = hashData(
self.portal_slap._getCacheComputerInformation(self.computer_id,
self.computer_id)
)
self.assertEqual(200, response.status)
self.assertTrue('last-modified' not in response.headers)
self.assertEqual(first_etag, response.headers.get('etag'))
self.assertEqual(first_body_fingerprint, hashData(response.body))
self.assertEqual(0, len(self.portal.portal_activities.getMessageList()))
# Trigger the computer reindexation
# This should trigger a new etag, but the body should be the same
self.computer.reindexObject()
self.commit()
# Second access
# Check that the result is stable, as the indexation timestamp is not changed yet
current_activity_count = len(self.portal.portal_activities.getMessageList())
response = self.portal_slap.getFullComputerInformation(self.computer_id)
self.commit()
self.assertEqual(200, response.status)
self.assertTrue('last-modified' not in response.headers)
self.assertEqual(first_etag, response.headers.get('etag'))
self.assertEqual(first_body_fingerprint, hashData(response.body))
self.assertEqual(current_activity_count, len(self.portal.portal_activities.getMessageList()))
self.tic()
# Third access, new calculation expected
# The retrieved informations comes from the cache
# But a new cache modification activity is triggered
response = self.portal_slap.getFullComputerInformation(self.computer_id)
self.commit()
self.assertEqual(200, response.status)
self.assertTrue('last-modified' not in response.headers)
second_etag = self.portal_slap._calculateRefreshEtag()
second_body_fingerprint = hashData(
self.portal_slap._getCacheComputerInformation(self.computer_id,
self.computer_id)
)
self.assertNotEqual(first_etag, second_etag)
# The indexation timestamp does not impact the response body
self.assertEqual(first_body_fingerprint, second_body_fingerprint)
self.assertEqual(first_etag, response.headers.get('etag'))
self.assertEqual(first_body_fingerprint, hashData(response.body))
self.assertEqual(1, len(self.portal.portal_activities.getMessageList()))
# Execute the cache modification activity
self.tic()
# 4th access
# The new etag value is now used
response = self.portal_slap.getFullComputerInformation(self.computer_id)
self.commit()
self.assertEqual(200, response.status)
self.assertTrue('last-modified' not in response.headers)
self.assertEqual(second_etag, response.headers.get('etag'))
self.assertEqual(first_body_fingerprint, hashData(response.body))
self.assertEqual(0, len(self.portal.portal_activities.getMessageList()))
# Edit the instance
# This should trigger a new etag and a new body
self.stop_requested_software_instance.edit(text_content=self.generateSafeXml())
self.commit()
# 5th access
# Check that the result is stable, as the indexation timestamp is not changed yet
current_activity_count = len(self.portal.portal_activities.getMessageList())
# Edition does not impact the etag
self.assertEqual(second_etag, self.portal_slap._calculateRefreshEtag())
third_body_fingerprint = hashData(
self.portal_slap._getCacheComputerInformation(self.computer_id,
self.computer_id)
)
# The edition impacts the response body
self.assertNotEqual(first_body_fingerprint, third_body_fingerprint)
response = self.portal_slap.getFullComputerInformation(self.computer_id)
self.commit()
self.assertEqual(200, response.status)
self.assertTrue('last-modified' not in response.headers)
self.assertEqual(second_etag, response.headers.get('etag'))
self.assertEqual(first_body_fingerprint, hashData(response.body))
self.assertEqual(current_activity_count, len(self.portal.portal_activities.getMessageList()))
self.tic()
# 6th, the instance edition triggered an interaction workflow
# which updated the cache
response = self.portal_slap.getFullComputerInformation(self.computer_id)
self.commit()
self.assertEqual(200, response.status)
self.assertTrue('last-modified' not in response.headers)
third_etag = self.portal_slap._calculateRefreshEtag()
self.assertNotEqual(second_etag, third_etag)
self.assertEqual(third_etag, response.headers.get('etag'))
self.assertEqual(third_body_fingerprint, hashData(response.body))
self.assertEqual(0, len(self.portal.portal_activities.getMessageList()))
# Remove the slave link to the partition
# Computer should loose permission to access the slave instance
self.start_requested_slave_instance.setAggregate('')
self.commit()
# 7th access
# Check that the result is stable, as the indexation timestamp is not changed yet
current_activity_count = len(self.portal.portal_activities.getMessageList())
# Edition does not impact the etag
self.assertEqual(third_etag, self.portal_slap._calculateRefreshEtag())
# The edition does not impact the response body yet, as the aggregate relation
# is not yet unindex
self.assertEqual(third_body_fingerprint, hashData(
self.portal_slap._getCacheComputerInformation(self.computer_id,
self.computer_id)
))
response = self.portal_slap.getFullComputerInformation(self.computer_id)
self.commit()
self.assertEqual(200, response.status)
self.assertTrue('last-modified' not in response.headers)
self.assertEqual(third_etag, response.headers.get('etag'))
self.assertEqual(third_body_fingerprint, hashData(response.body))
self.assertEqual(current_activity_count, len(self.portal.portal_activities.getMessageList()))
self.tic()
# 8th access
# changing the aggregate relation trigger the partition reindexation
# which trigger cache modification activity
# So, we should get the correct cached value
response = self.portal_slap.getFullComputerInformation(self.computer_id)
self.commit()
self.assertEqual(200, response.status)
self.assertTrue('last-modified' not in response.headers)
fourth_etag = self.portal_slap._calculateRefreshEtag()
fourth_body_fingerprint = hashData(
self.portal_slap._getCacheComputerInformation(self.computer_id,
self.computer_id)
)
self.assertNotEqual(third_etag, fourth_etag)
# The indexation timestamp does not impact the response body
self.assertNotEqual(third_body_fingerprint, fourth_body_fingerprint)
self.assertEqual(fourth_etag, response.headers.get('etag'))
self.assertEqual(fourth_body_fingerprint, hashData(response.body))
self.assertEqual(0, len(self.portal.portal_activities.getMessageList()))
class TestSlapOSSlapToolComputerAccess(TestSlapOSSlapToolMixin): class TestSlapOSSlapToolComputerAccess(TestSlapOSSlapToolMixin):
def test_getFullComputerInformation(self): def test_getFullComputerInformation(self):
self._makeComplexComputer(with_slave=True) self._makeComplexComputer(with_slave=True)
...@@ -81,7 +248,7 @@ class TestSlapOSSlapToolComputerAccess(TestSlapOSSlapToolMixin): ...@@ -81,7 +248,7 @@ class TestSlapOSSlapToolComputerAccess(TestSlapOSSlapToolMixin):
response.headers.get('cache-control')) response.headers.get('cache-control'))
self.assertEqual('REMOTE_USER', self.assertEqual('REMOTE_USER',
response.headers.get('vary')) response.headers.get('vary'))
self.assertTrue('last-modified' in response.headers) self.assertFalse('etag' in response.headers)
self.assertEqual('text/xml; charset=utf-8', self.assertEqual('text/xml; charset=utf-8',
response.headers.get('content-type')) response.headers.get('content-type'))
...@@ -992,7 +1159,7 @@ class TestSlapOSSlapToolInstanceAccess(TestSlapOSSlapToolMixin): ...@@ -992,7 +1159,7 @@ class TestSlapOSSlapToolInstanceAccess(TestSlapOSSlapToolMixin):
response.headers.get('cache-control')) response.headers.get('cache-control'))
self.assertEqual('REMOTE_USER', self.assertEqual('REMOTE_USER',
response.headers.get('vary')) response.headers.get('vary'))
self.assertTrue('last-modified' in response.headers) self.assertFalse('etag' in response.headers)
self.assertEqual('text/xml; charset=utf-8', self.assertEqual('text/xml; charset=utf-8',
response.headers.get('content-type')) response.headers.get('content-type'))
# check returned XML # check returned XML
......
...@@ -163,6 +163,8 @@ class SlapTool(BaseTool): ...@@ -163,6 +163,8 @@ class SlapTool(BaseTool):
#################################################### ####################################################
def _isTestRun(self): def _isTestRun(self):
if self.REQUEST.get('disable_isTestRun', False):
return False
if issubclass(self.getPortalObject().MailHost.__class__, DummyMailHostMixin) \ if issubclass(self.getPortalObject().MailHost.__class__, DummyMailHostMixin) \
or self.REQUEST.get('test_list'): or self.REQUEST.get('test_list'):
return True return True
...@@ -199,6 +201,7 @@ class SlapTool(BaseTool): ...@@ -199,6 +201,7 @@ class SlapTool(BaseTool):
self._getCachePlugin().set(key, DEFAULT_CACHE_SCOPE, self._getCachePlugin().set(key, DEFAULT_CACHE_SCOPE,
dict ( dict (
time=time.time(), time=time.time(),
refresh_etag=self._calculateRefreshEtag(),
data=self._getCacheComputerInformation(computer_id, user), data=self._getCacheComputerInformation(computer_id, user),
), ),
cache_duration=self.getPortalObject().portal_caches\ cache_duration=self.getPortalObject().portal_caches\
...@@ -273,8 +276,23 @@ class SlapTool(BaseTool): ...@@ -273,8 +276,23 @@ class SlapTool(BaseTool):
) )
) )
def _getComputerInformation(self, computer_id, user): def _calculateRefreshEtag(self):
user_document = _assertACI(self.getPortalObject().portal_catalog.unrestrictedGetResultValue( # check max indexation timestamp
# it is unlikely to get an empty catalog
last_indexed_entry = self.getPortalObject().portal_catalog(
select_list=['indexation_timestamp'],
portal_type=['Computer', 'Computer Partition',
'Software Instance', 'Slave Instance',
'Software Installation'],
sort_on=[('indexation_timestamp', 'DESC')],
limit=1,
)[0]
return '%s_%s' % (last_indexed_entry.uid,
last_indexed_entry.indexation_timestamp)
def _getComputerInformation(self, computer_id, user, refresh_etag):
portal = self.getPortalObject()
user_document = _assertACI(portal.portal_catalog.unrestrictedGetResultValue(
reference=user, portal_type=['Person', 'Computer', 'Software Instance'])) reference=user, portal_type=['Person', 'Computer', 'Software Instance']))
user_type = user_document.getPortalType() user_type = user_document.getPortalType()
self.REQUEST.response.setHeader('Content-Type', 'text/xml; charset=utf-8') self.REQUEST.response.setHeader('Content-Type', 'text/xml; charset=utf-8')
...@@ -285,21 +303,26 @@ class SlapTool(BaseTool): ...@@ -285,21 +303,26 @@ class SlapTool(BaseTool):
if user_type in ('Computer', 'Person'): if user_type in ('Computer', 'Person'):
if not self._isTestRun(): if not self._isTestRun():
cache_plugin = self._getCachePlugin() cache_plugin = self._getCachePlugin()
key = '%s_%s' % (computer_id, user)
try: try:
key = '%s_%s' % (computer_id, user)
entry = cache_plugin.get(key, DEFAULT_CACHE_SCOPE) entry = cache_plugin.get(key, DEFAULT_CACHE_SCOPE)
except KeyError: except KeyError:
entry = None entry = None
if entry is not None and isinstance(entry.getValue(), dict): if entry is not None and isinstance(entry.getValue(), dict):
result = entry.getValue()['data'] cached_dict = entry.getValue()
self._activateFillComputerInformationCache(computer_id, user) cached_etag = cached_dict.get('refresh_etag', None)
return result if (refresh_etag != cached_etag):
# Do not recalculate the computer information
# if nothing changed
self._activateFillComputerInformationCache(computer_id, user)
return cached_dict['data'], cached_etag
else: else:
self._activateFillComputerInformationCache(computer_id, user) self._activateFillComputerInformationCache(computer_id, user)
self.REQUEST.response.setStatus(503) self.REQUEST.response.setStatus(503)
return self.REQUEST.response return self.REQUEST.response, None
else: else:
return self._getCacheComputerInformation(computer_id, user) return self._getCacheComputerInformation(computer_id, user), None
else: else:
slap_computer._software_release_list = [] slap_computer._software_release_list = []
...@@ -317,7 +340,7 @@ class SlapTool(BaseTool): ...@@ -317,7 +340,7 @@ class SlapTool(BaseTool):
portal_type="Computer Partition") portal_type="Computer Partition")
self._calculateSlapComputerInformation(slap_computer, computer_partition_list) self._calculateSlapComputerInformation(slap_computer, computer_partition_list)
return dumps(slap_computer) return dumps(slap_computer), None
@UnrestrictedMethod @UnrestrictedMethod
def _getHostingSubscriptionIpList(self, computer_id, computer_partition_id): def _getHostingSubscriptionIpList(self, computer_id, computer_partition_id):
...@@ -358,7 +381,8 @@ class SlapTool(BaseTool): ...@@ -358,7 +381,8 @@ class SlapTool(BaseTool):
user = self.getPortalObject().portal_membership.getAuthenticatedMember().getUserName() user = self.getPortalObject().portal_membership.getAuthenticatedMember().getUserName()
if str(user) == computer_id: if str(user) == computer_id:
self._logAccess(user, user, '#access %s' % computer_id) self._logAccess(user, user, '#access %s' % computer_id)
result = self._getComputerInformation(computer_id, user) refresh_etag = self._calculateRefreshEtag()
body, etag = self._getComputerInformation(computer_id, user, refresh_etag)
if self.REQUEST.response.getStatus() == 200: if self.REQUEST.response.getStatus() == 200:
# Keep in cache server for 7 days # Keep in cache server for 7 days
...@@ -366,11 +390,12 @@ class SlapTool(BaseTool): ...@@ -366,11 +390,12 @@ class SlapTool(BaseTool):
'public, max-age=1, stale-if-error=604800') 'public, max-age=1, stale-if-error=604800')
self.REQUEST.response.setHeader('Vary', self.REQUEST.response.setHeader('Vary',
'REMOTE_USER') 'REMOTE_USER')
self.REQUEST.response.setHeader('Last-Modified', rfc1123_date(DateTime())) if etag is not None:
self.REQUEST.response.setBody(result) self.REQUEST.response.setHeader('Etag', etag)
self.REQUEST.response.setBody(body)
return self.REQUEST.response return self.REQUEST.response
else: else:
return result return body
security.declareProtected(Permissions.AccessContentsInformation, security.declareProtected(Permissions.AccessContentsInformation,
'getHostingSubscriptionIpList') 'getHostingSubscriptionIpList')
......
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="InteractionDefinition" module="Products.ERP5.Interaction"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>actbox_category</string> </key>
<value> <string>workflow</string> </value>
</item>
<item>
<key> <string>actbox_name</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>actbox_url</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>activate_script_name</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>after_script_name</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>before_commit_script_name</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>guard</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Instance_setAggregate</string> </value>
</item>
<item>
<key> <string>method_id</string> </key>
<value>
<list>
<string>_setAggregate.*</string>
</list>
</value>
</item>
<item>
<key> <string>once_per_transaction</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>portal_type_filter</string> </key>
<value>
<list>
<string>Slave Instance</string>
<string>Software Installation</string>
</list>
</value>
</item>
<item>
<key> <string>portal_type_group_filter</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>script_name</string> </key>
<value>
<list>
<string>Instance_reindexComputerPartition</string>
</list>
</value>
</item>
<item>
<key> <string>temporary_document_disallowed</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>trigger_type</string> </key>
<value> <int>2</int> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>Instance_reindexComputerPartition</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
<value> <string>SlapOSSlapTool</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>Instance_reindexComputerPartition</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment