slapos_cloud: Introduce SlapOSComputeNodeMixin

   Move away part of SlapTool code into a Mixin specific to fill and get Computer Information cache

master/bt5/slapos_cloud/MixinTemplateItem/portal_components/mixin.erp5.SlapOSComputeNodeMixin.py

+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Copyright (c) 2010-2022 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly advised to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from Products.ERP5Type.Cache import DEFAULT_CACHE_SCOPE
+from AccessControl import Unauthorized
+from AccessControl.Permissions import access_contents_information
+from AccessControl import getSecurityManager
+from Products.ERP5Type.UnrestrictedMethod import UnrestrictedMethod
+from Products.ERP5Type.tests.utils import DummyMailHostMixin
+
+import time
+from lxml import etree
+from zLOG import LOG, INFO
+
+try:
+  from slapos.slap.slap import (
+    Computer as ComputeNode,
+    ComputerPartition as SlapComputePartition,
+    SoftwareRelease)
+  from slapos.util import xml2dict, dumps
+except ImportError:
+  # Do no prevent instance from starting
+  # if libs are not installed
+  class ComputeNode:
+    def __init__(self):
+      raise ImportError
+  class SlapComputePartition:
+    def __init__(self):
+      raise ImportError
+  class SoftwareRelease:
+    def __init__(self):
+      raise ImportError
+  def xml2dict(dictionary):
+    raise ImportError
+  def dumps(*args):
+    raise ImportError
+
+
+def _assertACI(document):
+  sm = getSecurityManager()
+  if sm.checkPermission(access_contents_information,
+      document):
+    return document
+  raise Unauthorized('User %r has no access to %r' % (sm.getUser(), document))
+
+class SlapOSComputeNodeMixin(object):
+
+  def _getCachePlugin(self):
+    return self.getPortalObject().portal_caches\
+      .getRamCacheRoot().get('compute_node_information_cache_factory')\
+      .getCachePluginList()[0]
+
+  @UnrestrictedMethod
+  def _getSoftwareReleaseValueList(self):
+    """Returns list of Software Releases documents for compute_node"""
+    portal = self.getPortalObject()
+    software_release_list = []
+    for software_installation in portal.portal_catalog.unrestrictedSearchResults(
+      portal_type='Software Installation',
+      default_aggregate_uid=self.getUid(),
+      validation_state='validated',
+      ):
+      software_installation = _assertACI(software_installation.getObject())
+      software_release_response = SoftwareRelease(
+          software_release=software_installation.getUrlString().decode('UTF-8'),
+          computer_guid=self.getReference().decode('UTF-8'))
+      if software_installation.getSlapState() == 'destroy_requested':
+        software_release_response._requested_state = 'destroyed'
+      else:
+        software_release_response._requested_state = 'available'
+
+      known_state = software_installation.getTextAccessStatus()
+      if known_state.startswith("#access"):
+        software_release_response._known_state = 'available'
+      elif known_state.startswith("#building"):
+        software_release_response._known_state = 'building'
+      else:
+        software_release_response._known_state = 'error'
+
+      software_release_list.append(software_release_response)
+    return software_release_list
+
+  def _getCacheComputeNodeInformation(self, user):
+    self.REQUEST.response.setHeader('Content-Type', 'text/xml; charset=utf-8')
+    slap_compute_node = ComputeNode(self.getReference().decode("UTF-8"))
+
+    slap_compute_node._computer_partition_list = []
+    slap_compute_node._software_release_list = self._getSoftwareReleaseValueList()
+
+    unrestrictedSearchResults = self.getPortalObject().portal_catalog.unrestrictedSearchResults
+
+    compute_partition_list = unrestrictedSearchResults(
+      parent_uid=self.getUid(),
+      validation_state="validated",
+      portal_type="Compute Partition"
+    )
+    self._calculateSlapComputeNodeInformation(slap_compute_node, compute_partition_list)
+
+    return dumps(slap_compute_node)
+
+  def _activateFillComputeNodeInformationCache(self, user):
+    tag = 'compute_node_information_cache_fill_%s_%s' % (self.getReference(), user)
+    if self.getPortalObject().portal_activities.countMessageWithTag(tag) == 0:
+      self.activate(activity='SQLQueue', priority=2, tag=tag)._fillComputeNodeInformationCache(user)
+
+
+  def _fillComputeNodeInformationCache(self, user):
+    key = '%s_%s' % (self.getReference(), user)
+    try:
+      self._getCachePlugin().set(key, DEFAULT_CACHE_SCOPE,
+        dict (
+          time=time.time(),
+          refresh_etag=self._calculateRefreshEtag(),
+          data=self._getCacheComputeNodeInformation(user),
+        ),
+        cache_duration=self.getPortalObject().portal_caches\
+            .getRamCacheRoot().get('compute_node_information_cache_factory'\
+              ).cache_duration
+        )
+    except (Unauthorized, IndexError):
+      # XXX: Unauthorized hack. Race condition of not ready setup delivery which provides
+      # security information shall not make this method fail, as it will be
+      # called later anyway
+      # Note: IndexError ignored, as it happend in case if full reindex is
+      # called on site
+      pass
+
+
+  def _calculateRefreshEtag(self):
+    # check max indexation timestamp
+    # it is unlikely to get an empty catalog
+    last_indexed_entry = self.getPortalObject().portal_catalog(
+      select_list=['indexation_timestamp'],
+      portal_type=['Compute Node', 'Compute Partition',
+                   'Software Instance', 'Slave Instance',
+                   'Software Installation'],
+      sort_on=[('indexation_timestamp', 'DESC')],
+      limit=1,
+    )[0]
+    return '%s_%s' % (last_indexed_entry.uid,
+                      last_indexed_entry.indexation_timestamp)
+
+  def _isTestRun(self):
+    if self.REQUEST.get('disable_isTestRun', False):
+      return False
+    if issubclass(self.getPortalObject().MailHost.__class__, DummyMailHostMixin) \
+        or self.REQUEST.get('test_list'):
+      return True
+    return False
+
+  def _getComputeNodeInformation(self, user, refresh_etag):
+    portal = self.getPortalObject()
+    user_document = _assertACI(portal.portal_catalog.unrestrictedGetResultValue(
+      reference=user, portal_type=['Person', 'Compute Node', 'Software Instance']))
+    user_type = user_document.getPortalType()
+    self.REQUEST.response.setHeader('Content-Type', 'text/xml; charset=utf-8')
+
+    slap_compute_node = ComputeNode(self.getReference().decode("UTF-8"))
+
+    slap_compute_node._computer_partition_list = []
+    if user_type in ('Compute Node', 'Person'):
+      if not self._isTestRun():
+        cache_plugin = self._getCachePlugin()
+        key = '%s_%s' % (self.getReference(), user)
+        try:
+          entry = cache_plugin.get(key, DEFAULT_CACHE_SCOPE)
+        except KeyError:
+          entry = None
+
+        if entry is not None and isinstance(entry.getValue(), dict):
+          cached_dict = entry.getValue()
+          cached_etag = cached_dict.get('refresh_etag', None)
+          if (refresh_etag != cached_etag):
+            # Do not recalculate the compute_node information
+            # if nothing changed
+            self._activateFillComputeNodeInformationCache(user)
+          return cached_dict['data'], cached_etag
+        else:
+          self._activateFillComputeNodeInformationCache(user)
+          self.REQUEST.response.setStatus(503)
+          return self.REQUEST.response, None
+      else:
+        return self._getCacheComputeNodeInformation(user), None
+    else:
+      slap_compute_node._software_release_list = []
+
+    if user_type == 'Software Instance':
+      compute_partition_list = self.contentValues(
+        portal_type="Compute Partition",
+        checked_permission="View")
+    else:
+      compute_partition_list = self.getPortalObject().portal_catalog.unrestrictedSearchResults(
+                    parent_uid=self.getUid(),
+                    validation_state="validated",
+                    portal_type="Compute Partition")
+
+    self._calculateSlapComputeNodeInformation(slap_compute_node, compute_partition_list)
+    return dumps(slap_compute_node), None
+
+  def _calculateSlapComputeNodeInformation(self, slap_compute_node, compute_partition_list):
+    if len(compute_partition_list) == 0:
+      return
+
+    unrestrictedSearchResults = self.getPortalObject().portal_catalog.unrestrictedSearchResults
+
+    compute_partition_uid_list = [x.uid for x in compute_partition_list]
+    grouped_software_instance_list = unrestrictedSearchResults(
+      portal_type="Software Instance",
+      default_aggregate_uid=compute_partition_uid_list,
+      validation_state="validated",
+      group_by_list=['default_aggregate_uid'],
+      select_list=['default_aggregate_uid', 'count(*)']
+    )
+    slave_software_instance_list = unrestrictedSearchResults(
+      default_aggregate_uid=compute_partition_uid_list,
+      portal_type='Slave Instance',
+      validation_state="validated",
+      select_list=['default_aggregate_uid'],
+      **{"slapos_item.slap_state": "start_requested"}
+    )
+
+    for compute_partition in compute_partition_list:
+      software_instance_list = [x for x in grouped_software_instance_list if (x.default_aggregate_uid == compute_partition.getUid())]
+      if (len(software_instance_list) == 1) and (software_instance_list[0]['count(*)'] > 1):
+        software_instance_list = software_instance_list + software_instance_list
+      slap_compute_node._computer_partition_list.append(
+        self._getSlapPartitionByPackingList(
+          _assertACI(compute_partition.getObject()),
+          software_instance_list,
+          [x for x in slave_software_instance_list if (x.default_aggregate_uid == compute_partition.getUid())]
+        )
+      )
+
+
+  def _instanceXmlToDict(self, xml):
+    result_dict = {}
+    try:
+      result_dict = xml2dict(xml)
+    except (etree.XMLSchemaError, etree.XMLSchemaParseError, # pylint: disable=catching-non-exception
+      etree.XMLSchemaValidateError, etree.XMLSyntaxError): # pylint: disable=catching-non-exception
+      LOG('SlapOSComputeNodeCacheMixin', INFO, 'Issue during parsing xml:', error=True)
+    return result_dict
+
+  def _getSlapPartitionByPackingList(self, compute_partition_document,
+                                     software_instance_list,
+                                     shared_instance_sql_list):
+    compute_node = compute_partition_document
+    while compute_node.getPortalType() != 'Compute Node':
+      compute_node = compute_node.getParentValue()
+    compute_node_id = compute_node.getReference().decode("UTF-8")
+    slap_partition = SlapComputePartition(compute_node_id,
+      compute_partition_document.getReference().decode("UTF-8"))
+
+    slap_partition._software_release_document = None
+    slap_partition._requested_state = 'destroyed'
+    slap_partition._need_modification = 0
+
+    software_instance = None
+
+    if compute_partition_document.getSlapState() == 'busy':
+      software_instance_count = len(software_instance_list)
+      if software_instance_count == 1:
+        software_instance = _assertACI(software_instance_list[0].getObject())
+      elif software_instance_count > 1:
+        # XXX do not prevent the system to work if one partition is broken
+        raise NotImplementedError, "Too many instances linked to %s" % \
+           compute_partition_document.getRelativeUrl()
+
+    if software_instance is not None:
+      state = software_instance.getSlapState()
+      if state == "stop_requested":
+        slap_partition._requested_state = 'stopped'
+      if state == "start_requested":
+        slap_partition._requested_state = 'started'
+      slap_partition._access_status = software_instance.getTextAccessStatus()
+
+      slap_partition._software_release_document = SoftwareRelease(
+            software_release=software_instance.getUrlString().decode("UTF-8"),
+            computer_guid=compute_node_id)
+
+      slap_partition._need_modification = 1
+
+      parameter_dict = software_instance._asParameterDict(
+        shared_instance_sql_list=shared_instance_sql_list
+      )
+      # software instance has to define an xml parameter
+      slap_partition._parameter_dict = self._instanceXmlToDict(
+        parameter_dict.pop('xml'))
+      slap_partition._connection_dict = self._instanceXmlToDict(
+        parameter_dict.pop('connection_xml'))
+      slap_partition._filter_dict = self._instanceXmlToDict(
+        parameter_dict.pop('filter_xml'))
+      slap_partition._instance_guid = parameter_dict.pop('instance_guid')
+      for slave_instance_dict in parameter_dict.get("slave_instance_list", []):
+        if slave_instance_dict.has_key("connection_xml"):
+          slave_instance_dict.update(self._instanceXmlToDict(
+            slave_instance_dict.pop("connection_xml")))
+        if slave_instance_dict.has_key("xml"):
+          slave_instance_dict.update(self._instanceXmlToDict(
+            slave_instance_dict.pop("xml")))
+      slap_partition._parameter_dict.update(parameter_dict)
+
+    return slap_partition

master/bt5/slapos_cloud/MixinTemplateItem/portal_components/mixin.erp5.SlapOSComputeNodeMixin.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Mixin Component" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_recorded_property_dict</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+        <item>
+            <key> <string>default_reference</string> </key>
+            <value> <string>SlapOSComputeNodeMixin</string> </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>mixin.erp5.SlapOSComputeNodeMixin</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Mixin Component</string> </value>
+        </item>
+        <item>
+            <key> <string>sid</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_error_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_warning_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>version</string> </key>
+            <value> <string>erp5</string> </value>
+        </item>
+        <item>
+            <key> <string>workflow_history</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary/>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>component_validation_workflow</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
+                    </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="4" aka="AAAAAAAAAAQ=">
+    <pickle>
+      <global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_log</string> </key>
+            <value>
+              <list>
+                <dictionary>
+                  <item>
+                      <key> <string>action</string> </key>
+                      <value> <string>validate</string> </value>
+                  </item>
+                  <item>
+                      <key> <string>error_message</string> </key>
+                      <value>
+                        <list>
+                          <persistent> <string encoding="base64">AAAAAAAAAAU=</string> </persistent>
+                        </list>
+                      </value>
+                  </item>
+                  <item>
+                      <key> <string>validation_state</string> </key>
+                      <value> <string>validated</string> </value>
+                  </item>
+                </dictionary>
+              </list>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="5" aka="AAAAAAAAAAU=">
+    <pickle>
+      <global name="Message" module="Products.ERP5Type.Message"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>default</string> </key>
+            <value> <string>Class ${reference} must be defined</string> </value>
+        </item>
+        <item>
+            <key> <string>domain</string> </key>
+            <value> <string>erp5_ui</string> </value>
+        </item>
+        <item>
+            <key> <string>mapping</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>reference</string> </key>
+                    <value> <string>SlapOSComputeNodeCacheMixin</string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+        <item>
+            <key> <string>message</string> </key>
+            <value> <string>Class ${reference} must be defined</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

master/bt5/slapos_cloud/PortalTypeTypeMixinTemplateItem/type_mixin.xml

 <type_mixin>
  <portal_type id="Compute Node">
   <item>SlapOSCacheMixin</item>
+  <item>SlapOSComputeNodeMixin</item>
  </portal_type>
  <portal_type id="Compute Partition">
   <item>SlapOSCacheMixin</item>

master/bt5/slapos_cloud/bt/template_mixin_id_list

-mixin.erp5.SlapOSCacheMixin
\ No newline at end of file
+mixin.erp5.SlapOSCacheMixin
+mixin.erp5.SlapOSComputeNodeMixin
\ No newline at end of file

master/bt5/slapos_cloud/bt/template_path_list

@@ -19,6 +19,8 @@ portal_alarms/slapos_stop_collect_instance
 portal_alarms/slapos_update_compute_node_capacity_scope
 portal_caches/access_status_data_cache_factory
 portal_caches/access_status_data_cache_factory/volatile_cache_plugin
+portal_caches/compute_node_information_cache_factory
+portal_caches/compute_node_information_cache_factory/persistent_cache_plugin
 portal_caches/last_stored_data_cache_factory
 portal_caches/last_stored_data_cache_factory/volatile_cache_plugin
 product_module/compute_node

master/bt5/slapos_cloud/bt/template_portal_type_type_mixin_list

 Compute Node | SlapOSCacheMixin
+Compute Node | SlapOSComputeNodeMixin
 Compute Partition | SlapOSCacheMixin
 Person | SlapOSCacheMixin
\ No newline at end of file

master/bt5/slapos_slap_tool/ExtensionTemplateItem/portal_components/extension.erp5.SlapOSSlapTool.py

@@ -17,30 +17,29 @@ def getComputeNodeReferenceAndUserId(item):
       compute_node = partition.getParentValue()
 
   if compute_node is not None and compute_node.getValidationState() == 'validated':
-    return compute_node.getReference(), compute_node.getUserId()
-  return None, None
+    return compute_node, compute_node.getReference(), compute_node.getUserId()
+  return None, None, None
 
 
 def Item_activateFillComputeNodeInformationCache(state_change):
   item = state_change['object']
   portal = item.getPortalObject()
-  compute_node_reference, user_id = getComputeNodeReferenceAndUserId(item)
-  if compute_node_reference is None:
+  compute_node, compute_node_reference, user_id = getComputeNodeReferenceAndUserId(item)
+  if compute_node is None:
     return None
 
   if user_id is None:
     return None
 
   user = portal.acl_users.getUserById(user_id)
-
   if user is None:
     raise ValueError("User %s not found" % user_id)
 
   sm = getSecurityManager()
   try:
     newSecurityManager(None, user)
-    portal.portal_slap._activateFillComputeNodeInformationCache(
-        compute_node_reference, compute_node_reference)
+    compute_node._activateFillComputeNodeInformationCache(
+      compute_node_reference)
   finally:
     setSecurityManager(sm)
 

master/bt5/slapos_slap_tool/TestTemplateItem/portal_components/test.erp5.testSlapOSSlapTool.py

@@ -78,16 +78,16 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
     self.tic()
 
     self.login(self.compute_node_user_id)
-
+    self.portal_slap.getFullComputerInformation(self.compute_node_id)
+    
     # First access.
     # Cache has been filled by interaction workflow
     # (luckily, it seems the cache is filled after everything is indexed)
     response = self.portal_slap.getFullComputerInformation(self.compute_node_id)
     self.commit()
-    first_etag = self.portal_slap._calculateRefreshEtag()
+    first_etag = self.compute_node._calculateRefreshEtag()
     first_body_fingerprint = hashData(
-      self.portal_slap._getCacheComputeNodeInformation(self.compute_node_id,
-                                                    self.compute_node_id)
+      self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
     )
     self.assertEqual(200, response.status)
     self.assertTrue('last-modified' not in response.headers)
@@ -120,10 +120,9 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
     self.commit()
     self.assertEqual(200, response.status)
     self.assertTrue('last-modified' not in response.headers)
-    second_etag = self.portal_slap._calculateRefreshEtag()
+    second_etag = self.compute_node._calculateRefreshEtag()
     second_body_fingerprint = hashData(
-      self.portal_slap._getCacheComputeNodeInformation(self.compute_node_id,
-                                                    self.compute_node_id)
+      self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
     )
     self.assertNotEqual(first_etag, second_etag)
     # The indexation timestamp does not impact the response body
@@ -154,10 +153,9 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
     # Check that the result is stable, as the indexation timestamp is not changed yet
     current_activity_count = len(self.portal.portal_activities.getMessageList())
     # Edition does not impact the etag
-    self.assertEqual(second_etag, self.portal_slap._calculateRefreshEtag())
+    self.assertEqual(second_etag, self.compute_node._calculateRefreshEtag())
     third_body_fingerprint = hashData(
-      self.portal_slap._getCacheComputeNodeInformation(self.compute_node_id,
-                                                    self.compute_node_id)
+      self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
     )
     # The edition impacts the response body
     self.assertNotEqual(first_body_fingerprint, third_body_fingerprint)
@@ -177,7 +175,7 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
     self.commit()
     self.assertEqual(200, response.status)
     self.assertTrue('last-modified' not in response.headers)
-    third_etag = self.portal_slap._calculateRefreshEtag()
+    third_etag = self.compute_node._calculateRefreshEtag()
     self.assertNotEqual(second_etag, third_etag)
     self.assertEqual(third_etag, response.headers.get('etag'))
     self.assertEqual(third_body_fingerprint, hashData(response.body))
@@ -192,12 +190,11 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
     # Check that the result is stable, as the indexation timestamp is not changed yet
     current_activity_count = len(self.portal.portal_activities.getMessageList())
     # Edition does not impact the etag
-    self.assertEqual(third_etag, self.portal_slap._calculateRefreshEtag())
+    self.assertEqual(third_etag, self.compute_node._calculateRefreshEtag())
     # The edition does not impact the response body yet, as the aggregate relation
     # is not yet unindex
     self.assertEqual(third_body_fingerprint, hashData(
-      self.portal_slap._getCacheComputeNodeInformation(self.compute_node_id,
-                                                    self.compute_node_id)
+      self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
     ))
     response = self.portal_slap.getFullComputerInformation(self.compute_node_id)
     self.commit()
@@ -217,10 +214,9 @@ class TestSlapOSSlapToolgetFullComputerInformation(TestSlapOSSlapToolMixin):
     self.commit()
     self.assertEqual(200, response.status)
     self.assertTrue('last-modified' not in response.headers)
-    fourth_etag = self.portal_slap._calculateRefreshEtag()
+    fourth_etag = self.compute_node._calculateRefreshEtag()
     fourth_body_fingerprint = hashData(
-      self.portal_slap._getCacheComputeNodeInformation(self.compute_node_id,
-                                                    self.compute_node_id)
+      self.compute_node._getCacheComputeNodeInformation(self.compute_node_id)
     )
     self.assertNotEqual(third_etag, fourth_etag)
     # The indexation timestamp does not impact the response body
@@ -733,43 +729,9 @@ class TestSlapOSSlapToolComputeNodeAccess(TestSlapOSSlapToolMixin):
       response = self.portal_slap.computerBang(self.compute_node_id,
         error_log)
       self.assertEqual('None', response)
-      created_at = rfc1123_date(DateTime())
-      since = created_at
-      response = self.portal_slap.getComputerStatus(self.compute_node_id)
-      # check returned XML
-      xml_fp = StringIO.StringIO()
-
-      xml.dom.ext.PrettyPrint(xml.dom.ext.reader.Sax.FromXml(response.body),
-          stream=xml_fp)
-      xml_fp.seek(0)
-      got_xml = xml_fp.read()
-      expected_xml = """\
-<?xml version='1.0' encoding='UTF-8'?>
-<marshal>
-  <dictionary id='i2'>
-    <string>created_at</string>
-    <string>%(created_at)s</string>
-    <string>no_data_since_15_minutes</string>
-    <int>0</int>
-    <string>no_data_since_5_minutes</string>
-    <int>0</int>
-    <string>since</string>
-    <string>%(since)s</string>
-    <string>state</string>
-    <string/>
-    <string>text</string>
-    <string>#error bang</string>
-    <string>user</string>
-    <string>%(compute_node_id)s</string>
-  </dictionary>
-</marshal>
-""" % dict(
-    created_at=created_at,
-    since=since,
-    compute_node_id=self.compute_node_id,
-  )
-      self.assertEqual(expected_xml, got_xml,
-          '\n'.join([q for q in difflib.unified_diff(expected_xml.split('\n'), got_xml.split('\n'))]))
+      # We do not assert getComputerStatus on this test, since
+      # the change of the timestamp is part of reportComputeNodeBang
+      
       self.assertComputeNodeBangSimulator((), {'comment': error_log})
     finally:
       if os.path.exists(self.compute_node_bang_simulator):
@@ -2635,43 +2597,9 @@ class TestSlapOSSlapToolPersonAccess(TestSlapOSSlapToolMixin):
       response = self.portal_slap.computerBang(self.compute_node_id,
         error_log)
       self.assertEqual('None', response)
-      created_at = rfc1123_date(DateTime())
-      since = created_at
-      response = self.portal_slap.getComputerStatus(self.compute_node_id)
-      # check returned XML
-      xml_fp = StringIO.StringIO()
-
-      xml.dom.ext.PrettyPrint(xml.dom.ext.reader.Sax.FromXml(response.body),
-          stream=xml_fp)
-      xml_fp.seek(0)
-      got_xml = xml_fp.read()
-      expected_xml = """\
-<?xml version='1.0' encoding='UTF-8'?>
-<marshal>
-  <dictionary id='i2'>
-    <string>created_at</string>
-    <string>%(created_at)s</string>
-    <string>no_data_since_15_minutes</string>
-    <int>0</int>
-    <string>no_data_since_5_minutes</string>
-    <int>0</int>
-    <string>since</string>
-    <string>%(since)s</string>
-    <string>state</string>
-    <string/>
-    <string>text</string>
-    <string>#error bang</string>
-    <string>user</string>
-    <string>%(person_reference)s</string>
-  </dictionary>
-</marshal>
-""" % dict(
-    created_at=created_at,
-    since=since,
-    person_reference=self.person_reference,
-  )
-      self.assertEqual(expected_xml, got_xml,
-          '\n'.join([q for q in difflib.unified_diff(expected_xml.split('\n'), got_xml.split('\n'))]))
+      # We do not assert getComputerStatus on this test, since
+      # the change of the timestamp is part of reportComputeNodeBang
+      
       self.assertComputeNodeBangSimulator((), {'comment': error_log})
     finally:
       if os.path.exists(self.compute_node_bang_simulator):

master/bt5/slapos_slap_tool/ToolComponentTemplateItem/portal_components/tool.erp5.SlapTool.py

@@ -38,11 +38,8 @@ from Products.DCWorkflow.DCWorkflow import ValidationFailed
 from Products.ERP5Type.Globals import InitializeClass
 from Products.ERP5Type.Tool.BaseTool import BaseTool
 from Products.ERP5Type import Permissions
-from Products.ERP5Type.Cache import DEFAULT_CACHE_SCOPE
 from Products.ERP5Type.Cache import CachingMethod
 from lxml import etree
-import time
-from Products.ERP5Type.tests.utils import DummyMailHostMixin
 try:
   from slapos.slap.slap import (
     Computer as ComputeNode,
@@ -162,167 +159,6 @@ class SlapTool(BaseTool):
   # Public GET methods
   ####################################################
 
-  def _isTestRun(self):
-    if self.REQUEST.get('disable_isTestRun', False):
-      return False
-    if issubclass(self.getPortalObject().MailHost.__class__, DummyMailHostMixin) \
-        or self.REQUEST.get('test_list'):
-      return True
-    return False
-
-  def _getCachePlugin(self):
-    return self.getPortalObject().portal_caches\
-      .getRamCacheRoot().get('compute_node_information_cache_factory')\
-      .getCachePluginList()[0]
-
-  def _getCacheComputeNodeInformation(self, compute_node_id, user):
-    self.REQUEST.response.setHeader('Content-Type', 'text/xml; charset=utf-8')
-    slap_compute_node = ComputeNode(compute_node_id.decode("UTF-8"))
-    parent_uid = self._getComputeNodeUidByReference(compute_node_id)
-
-    slap_compute_node._computer_partition_list = []
-    slap_compute_node._software_release_list = \
-       self._getSoftwareReleaseValueListForComputeNode(compute_node_id)
-
-    unrestrictedSearchResults = self.getPortalObject().portal_catalog.unrestrictedSearchResults
-
-    compute_partition_list = unrestrictedSearchResults(
-      parent_uid=parent_uid,
-      validation_state="validated",
-      portal_type="Compute Partition"
-    )
-    self._calculateSlapComputeNodeInformation(slap_compute_node, compute_partition_list)
-
-    return dumps(slap_compute_node)
-
-  def _fillComputeNodeInformationCache(self, compute_node_id, user):
-    key = '%s_%s' % (compute_node_id, user)
-    try:
-      self._getCachePlugin().set(key, DEFAULT_CACHE_SCOPE,
-        dict (
-          time=time.time(),
-          refresh_etag=self._calculateRefreshEtag(),
-          data=self._getCacheComputeNodeInformation(compute_node_id, user),
-        ),
-        cache_duration=self.getPortalObject().portal_caches\
-            .getRamCacheRoot().get('compute_node_information_cache_factory'\
-              ).cache_duration
-        )
-    except (Unauthorized, IndexError):
-      # XXX: Unauthorized hack. Race condition of not ready setup delivery which provides
-      # security information shall not make this method fail, as it will be
-      # called later anyway
-      # Note: IndexError ignored, as it happend in case if full reindex is
-      # called on site
-      pass
-
-
-  def _activateFillComputeNodeInformationCache(self, compute_node_id, user):
-    tag = 'compute_node_information_cache_fill_%s_%s' % (compute_node_id, user)
-    if self.getPortalObject().portal_activities.countMessageWithTag(tag) == 0:
-      self.activate(activity='SQLQueue', tag=tag)._fillComputeNodeInformationCache(
-        compute_node_id, user)
-
-  def _calculateSlapComputeNodeInformation(self, slap_compute_node, compute_partition_list):
-    if len(compute_partition_list) == 0:
-      return
-
-    unrestrictedSearchResults = self.getPortalObject().portal_catalog.unrestrictedSearchResults
-
-    compute_partition_uid_list = [x.uid for x in compute_partition_list]
-    grouped_software_instance_list = unrestrictedSearchResults(
-      portal_type="Software Instance",
-      default_aggregate_uid=compute_partition_uid_list,
-      validation_state="validated",
-      group_by_list=['default_aggregate_uid'],
-      select_list=['default_aggregate_uid', 'count(*)']
-    )
-    slave_software_instance_list = unrestrictedSearchResults(
-      default_aggregate_uid=compute_partition_uid_list,
-      portal_type='Slave Instance',
-      validation_state="validated",
-      select_list=['default_aggregate_uid'],
-      **{"slapos_item.slap_state": "start_requested"}
-    )
-
-    for compute_partition in compute_partition_list:
-      software_instance_list = [x for x in grouped_software_instance_list if (x.default_aggregate_uid == compute_partition.getUid())]
-      if (len(software_instance_list) == 1) and (software_instance_list[0]['count(*)'] > 1):
-        software_instance_list = software_instance_list + software_instance_list
-      slap_compute_node._computer_partition_list.append(
-        self._getSlapPartitionByPackingList(
-          _assertACI(compute_partition.getObject()),
-          software_instance_list,
-          [x for x in slave_software_instance_list if (x.default_aggregate_uid == compute_partition.getUid())]
-        )
-      )
-
-  def _calculateRefreshEtag(self):
-    # check max indexation timestamp
-    # it is unlikely to get an empty catalog
-    last_indexed_entry = self.getPortalObject().portal_catalog(
-      select_list=['indexation_timestamp'],
-      portal_type=['Compute Node', 'Compute Partition',
-                   'Software Instance', 'Slave Instance',
-                   'Software Installation'],
-      sort_on=[('indexation_timestamp', 'DESC')],
-      limit=1,
-    )[0]
-    return '%s_%s' % (last_indexed_entry.uid,
-                      last_indexed_entry.indexation_timestamp)
-
-  def _getComputeNodeInformation(self, compute_node_id, user, refresh_etag):
-    portal = self.getPortalObject()
-    user_document = _assertACI(portal.portal_catalog.unrestrictedGetResultValue(
-      reference=user, portal_type=['Person', 'Compute Node', 'Software Instance']))
-    user_type = user_document.getPortalType()
-    self.REQUEST.response.setHeader('Content-Type', 'text/xml; charset=utf-8')
-    slap_compute_node = ComputeNode(compute_node_id.decode("UTF-8"))
-    parent_uid = self._getComputeNodeUidByReference(compute_node_id)
-
-    slap_compute_node._computer_partition_list = []
-    if user_type in ('Compute Node', 'Person'):
-      if not self._isTestRun():
-        cache_plugin = self._getCachePlugin()
-        key = '%s_%s' % (compute_node_id, user)
-        try:
-          entry = cache_plugin.get(key, DEFAULT_CACHE_SCOPE)
-        except KeyError:
-          entry = None
-
-        if entry is not None and isinstance(entry.getValue(), dict):
-          cached_dict = entry.getValue()
-          cached_etag = cached_dict.get('refresh_etag', None)
-          if (refresh_etag != cached_etag):
-            # Do not recalculate the compute_node information
-            # if nothing changed
-            self._activateFillComputeNodeInformationCache(compute_node_id, user)
-          return cached_dict['data'], cached_etag
-        else:
-          self._activateFillComputeNodeInformationCache(compute_node_id, user)
-          self.REQUEST.response.setStatus(503)
-          return self.REQUEST.response, None
-      else:
-        return self._getCacheComputeNodeInformation(compute_node_id, user), None
-    else:
-      slap_compute_node._software_release_list = []
-
-    if user_type == 'Software Instance':
-      compute_node = self.getPortalObject().portal_catalog.unrestrictedSearchResults(
-        portal_type='Compute Node', reference=compute_node_id,
-        validation_state="validated")[0].getObject()
-      compute_partition_list = compute_node.contentValues(
-        portal_type="Compute Partition",
-        checked_permission="View")
-    else:
-      compute_partition_list = self.getPortalObject().portal_catalog.unrestrictedSearchResults(
-                    parent_uid=parent_uid,
-                    validation_state="validated",
-                    portal_type="Compute Partition")
-
-    self._calculateSlapComputeNodeInformation(slap_compute_node, compute_partition_list)
-    return dumps(slap_compute_node), None
-
   @UnrestrictedMethod
   def _getInstanceTreeIpList(self, compute_node_id, compute_partition_id):
     software_instance = self._getSoftwareInstanceForComputePartition(
@@ -363,8 +199,11 @@ class SlapTool(BaseTool):
     if str(user) == computer_id:
       compute_node = self.getPortalObject().portal_membership.getAuthenticatedMember().getUserValue()
       compute_node.setAccessStatus('#access %s' % computer_id)
-    refresh_etag = self._calculateRefreshEtag()
-    body, etag = self._getComputeNodeInformation(computer_id, user, refresh_etag)
+    else:
+      compute_node = self._getComputeNodeDocument(computer_id)
+
+    refresh_etag = compute_node._calculateRefreshEtag()
+    body, etag = compute_node._getComputeNodeInformation(user, refresh_etag)
 
     if self.REQUEST.response.getStatus() == 200:
       # Keep in cache server for 7 days
@@ -1576,37 +1415,6 @@ class SlapTool(BaseTool):
       'timestamp': "%i" % timestamp,
     }
 
-  @UnrestrictedMethod
-  def _getSoftwareReleaseValueListForComputeNode(self, compute_node_reference):
-    """Returns list of Software Releases documentsfor compute_node"""
-    compute_node_document = self._getComputeNodeDocument(compute_node_reference)
-    portal = self.getPortalObject()
-    software_release_list = []
-    for software_installation in portal.portal_catalog.unrestrictedSearchResults(
-      portal_type='Software Installation',
-      default_aggregate_uid=compute_node_document.getUid(),
-      validation_state='validated',
-      ):
-      software_installation = _assertACI(software_installation.getObject())
-      software_release_response = SoftwareRelease(
-          software_release=software_installation.getUrlString().decode('UTF-8'),
-          computer_guid=compute_node_reference.decode('UTF-8'))
-      if software_installation.getSlapState() == 'destroy_requested':
-        software_release_response._requested_state = 'destroyed'
-      else:
-        software_release_response._requested_state = 'available'
-
-      known_state = software_installation.getTextAccessStatus()
-      if known_state.startswith("#access"):
-        software_release_response._known_state = 'available'
-      elif known_state.startswith("#building"):
-        software_release_response._known_state = 'building'
-      else:
-        software_release_response._known_state = 'error'
-
-      software_release_list.append(software_release_response)
-    return software_release_list
-
   @convertToREST
   def _softwareReleaseError(self, url, compute_node_id, error_log):
     """

master/bt5/slapos_slap_tool/bt/template_keep_path_list

 portal_caches/last_stored_data_cache_factory
-portal_caches/last_stored_data_cache_factory/volatile_cache_plugin
\ No newline at end of file
+portal_caches/last_stored_data_cache_factory/volatile_cache_plugin
+portal_caches/compute_node_information_cache_factory
+portal_caches/compute_node_information_cache_factory/persistent_cache_plugin
\ No newline at end of file

master/bt5/slapos_slap_tool/bt/template_path_list

-portal_caches/compute_node_information_cache_factory
-portal_caches/compute_node_information_cache_factory/persistent_cache_plugin
 portal_caches/slap_cache_factory
 portal_caches/slap_cache_factory/persistent_cache_plugin
 web_site_module/slapos_hateoas