From 5e966f018366e598a7b43c07ec58c166835440b6 Mon Sep 17 00:00:00 2001
From: Ivan Tyagov <ivan@nexedi.com>
Date: Thu, 18 Oct 2012 17:24:37 +0300
Subject: [PATCH] Add Cache Bag implementation (multi level caching). Fix tests
 (DMS) to use by default Cache Bags. Fix some dead code and clean up. Move
 cache initialization away from DMS into Cache Factory.

---
 product/ERP5/mixin/cached_convertable.py      | 68 ++++++++--------
 product/ERP5OOo/tests/testDms.py              |  7 +-
 product/ERP5OOo/tests/testDmsWithFlare.py     |  9 ++-
 .../ERP5OOo/tests/testDmsWithPreConversion.py |  1 -
 product/ERP5Type/Core/CacheBag.py             | 77 +++++++++++++++++++
 product/ERP5Type/Core/CacheFactory.py         | 12 ++-
 product/ERP5Type/Tool/CacheTool.py            |  2 +-
 7 files changed, 130 insertions(+), 46 deletions(-)
 create mode 100644 product/ERP5Type/Core/CacheBag.py

diff --git a/product/ERP5/mixin/cached_convertable.py b/product/ERP5/mixin/cached_convertable.py
index ebda3ff7c5..e27aeaefae 100644
--- a/product/ERP5/mixin/cached_convertable.py
+++ b/product/ERP5/mixin/cached_convertable.py
@@ -74,20 +74,15 @@ class CachedConvertableMixin:
   def _getCacheFactory(self):
     """
     """
+    # XXX: is this really needed ?
     if self.getOriginalDocument() is None:
       return None
+
     portal = self.getPortalObject()
-    cache_tool = portal.portal_caches
-    preference_tool = portal.portal_preferences
-    cache_factory_name = preference_tool.getPreferredConversionCacheFactory('document_cache_factory')
-    cache_factory = cache_tool.getRamCacheRoot().get(cache_factory_name)
-    #XXX This conditional statement should be remove as soon as
-    #Broadcasting will be enable among all zeo clients.
-    #Interaction which update portal_caches should interact with all nodes.
-    if cache_factory is None and getattr(cache_tool, cache_factory_name, None) is not None:
-      #ram_cache_root is not up to date for current node
-      cache_tool.updateCache()
-    return cache_tool.getRamCacheRoot().get(cache_factory_name)
+    cache_factory_name = portal.portal_preferences.getPreferredConversionCacheFactory('document_cache_factory')
+    if cache_factory_name is not None:
+      return getattr(portal.portal_caches, cache_factory_name, None)
+
 
   security.declareProtected(Permissions.AccessContentsInformation,
                                                              'generateCacheId')
@@ -167,23 +162,18 @@ class CachedConvertableMixin:
         self.temp_conversion_data = {}
       self.temp_conversion_data[cache_id] = stored_data_dict
       return
-    cache_duration = cache_factory.cache_duration
     # The purpose of this transaction cache is to help calls
     # to the same cache value in the same transaction.
     tv = getTransactionalVariable()
     tv[cache_id] = stored_data_dict
-    for cache_plugin in cache_factory.getCachePluginList():
-      cache_plugin.set(cache_id, DEFAULT_CACHE_SCOPE,
-                       stored_data_dict, cache_duration=cache_duration)
+    cache_factory.set(cache_id, stored_data_dict)
 
   security.declareProtected(Permissions.View, '_getConversionDataDict')
   def _getConversionDataDict(self, **kw):
     """
     """
     cache_id = self._getCacheKey(**kw)
-    cache_factory = self._getCacheFactory()
-    if cache_factory is None:
-      return getattr(aq_base(self), 'temp_conversion_data', {})[cache_id]
+
     # The purpose of this cache is to help calls to the same cache value
     # in the same transaction.
     tv = getTransactionalVariable()
@@ -191,26 +181,28 @@ class CachedConvertableMixin:
       return tv[cache_id]
     except KeyError:
       pass
-    for cache_plugin in cache_factory.getCachePluginList():
-      cache_entry = cache_plugin.get(cache_id, DEFAULT_CACHE_SCOPE)
-      if cache_entry is not None:
-        data_dict = cache_entry.getValue()
-        if data_dict:
-          if isinstance(data_dict, tuple):
-            # Backward compatibility: if cached value is a tuple
-            # as it was before refactoring
-            # http://svn.erp5.org?rev=35216&view=rev
-            # raise a KeyError to invalidate this cache entry and force
-            # calculation of a new conversion
-            raise KeyError('Old cache conversion format,'\
-                               'cache entry invalidated for key:%r' % cache_id)
-          content_md5 = data_dict['content_md5']
-          if content_md5 != self.getContentMd5():
-            raise KeyError, 'Conversion cache key is compromised for %r' % cache_id
-          # Fill transactional cache in order to help
-          # querying real cache during same transaction
-          tv[cache_id] = data_dict
-          return data_dict
+
+    # get preferred cache factory or cache bag
+    cache_factory = self._getCacheFactory()
+    if cache_factory is not None:
+      data_dict = cache_factory.get(cache_id, None)
+      if data_dict:
+        if isinstance(data_dict, tuple):
+          # Backward compatibility: if cached value is a tuple
+          # as it was before refactoring
+          # http://svn.erp5.org?rev=35216&view=rev
+          # raise a KeyError to invalidate this cache entry and force
+          # calculation of a new conversion
+          raise KeyError('Old cache conversion format,'\
+                       'cache entry invalidated for key:%r' % cache_id)
+        content_md5 = data_dict['content_md5']
+        if content_md5 != self.getContentMd5():
+          raise KeyError, 'Conversion cache key is compromised for %r' % cache_id
+        # Fill transactional cache in order to help
+        # querying real cache during same transaction
+        tv[cache_id] = data_dict
+        return data_dict
+
     raise KeyError, 'Conversion cache key does not exists for %r' % cache_id
 
   security.declareProtected(Permissions.View, 'getConversion')
diff --git a/product/ERP5OOo/tests/testDms.py b/product/ERP5OOo/tests/testDms.py
index 6c9d7a56da..1f857dec03 100644
--- a/product/ERP5OOo/tests/testDms.py
+++ b/product/ERP5OOo/tests/testDms.py
@@ -133,9 +133,14 @@ class TestDocumentMixin(ERP5TypeTestCase):
     preference_list = self.portal.portal_preferences.contentValues(
                                                        portal_type=portal_type)
     if not preference_list:
+      # create a Cache Bag for tests
+      cache_bag = self.portal.portal_caches.newContent(portal_type = 'Cache Bag')
+      cache_bag.cache_duration = 36000
+      cache_plugin = cache_bag.newContent(portal_type='Ram Cache')
+      cache_plugin.cache_expire_check_interval = 54000
       preference = self.portal.portal_preferences.newContent(title="Default System Preference",
                                                              # use local RAM based cache as some tests need it
-                                                             preferred_conversion_cache_factory = 'erp5_content_long',
+                                                             preferred_conversion_cache_factory = cache_bag.getId(),
                                                              portal_type=portal_type)
     else:
       preference = preference_list[0]
diff --git a/product/ERP5OOo/tests/testDmsWithFlare.py b/product/ERP5OOo/tests/testDmsWithFlare.py
index 2ff6e39eed..4b226160f8 100644
--- a/product/ERP5OOo/tests/testDmsWithFlare.py
+++ b/product/ERP5OOo/tests/testDmsWithFlare.py
@@ -44,11 +44,14 @@ class TestDocumentWithFlare(TestDocument):
   def setSystemPreference(self):
     system_preference = TestDocument.setSystemPreference(self)
     memcached = _getPersistentMemcachedServerDict()
-    system_preference.setPreferredConversionCacheFactory('dms_cache_factory')
+    # create a Cache Bag for tests
+    cache_bag = self.portal.portal_caches.newContent(portal_type = 'Cache Bag')
+    cache_bag.cache_duration = 15768000
+    cache_plugin = cache_bag.newContent(portal_type='Distributed Ram Cache')
+    system_preference.setPreferredConversionCacheFactory(cache_bag.getId())
     persistent_memcached_plugin = self.portal.portal_memcached.persistent_memcached_plugin
     persistent_memcached_plugin.setUrlString('%s:%s' %(memcached['hostname'], memcached['port']))
-    self.portal.portal_caches.dms_cache_factory.persistent_cache_plugin.setSpecialiseValue(persistent_memcached_plugin)
-
+    cache_plugin.setSpecialiseValue(persistent_memcached_plugin)
 
 def test_suite():
   suite = unittest.TestSuite()
diff --git a/product/ERP5OOo/tests/testDmsWithPreConversion.py b/product/ERP5OOo/tests/testDmsWithPreConversion.py
index 2a96513c7d..d9487645f7 100644
--- a/product/ERP5OOo/tests/testDmsWithPreConversion.py
+++ b/product/ERP5OOo/tests/testDmsWithPreConversion.py
@@ -68,7 +68,6 @@ class TestDocumentWithPreConversion(TestDocument):
     """
       Test pre converion only happens on proper documents.
     """
-    print "da"
     image = self.portal.image_module.newContent(portal_type='Image',
                                                reference='Embedded-XXX',
                                                version='001',
diff --git a/product/ERP5Type/Core/CacheBag.py b/product/ERP5Type/Core/CacheBag.py
new file mode 100644
index 0000000000..3ec6230804
--- /dev/null
+++ b/product/ERP5Type/Core/CacheBag.py
@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Copyright (c) 2005 Nexedi SARL and Contributors. All Rights Reserved.
+#                     Ivan Tyagov <ivan@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+
+from AccessControl import ClassSecurityInfo
+from Products.ERP5Type import Permissions
+from Products.ERP5Type.Cache import DEFAULT_CACHE_SCOPE
+from Products.ERP5Type.Core.CacheFactory import CacheFactory
+
+class CacheBag(CacheFactory):
+  """
+  CacheBag is a special type of CacheFactory that allows multi level caching
+  in different backends describe by CachePlugin.
+
+  CacheBag 1
+     - Cache Plugin 1 (priority 0)
+     - Cache Plugin 2 (priority 1)
+  """
+
+  meta_type = 'ERP5 Cache Bag'
+  portal_type = 'Cache Bag'
+
+  security = ClassSecurityInfo()
+
+  security.declareProtected(Permissions.AccessContentsInformation, 'get')
+  def get(self, cache_id, default=None):
+    """
+      Get value or return default.
+    """
+    ram_cache_factory_plugin_list = self.getRamCacheFactoryPluginList()
+
+    for cache_plugin in ram_cache_factory_plugin_list:
+      data_dict = cache_plugin.get(cache_id, DEFAULT_CACHE_SCOPE, default)
+      if data_dict is not None:
+        value = data_dict.getValue()
+        if ram_cache_factory_plugin_list.index(cache_plugin)>0:
+          # update first plugin as it's the one to be used
+          cache_duration = self.getRamCacheFactory().cache_duration
+          ram_cache_factory_plugin_list[0].set(cache_id, DEFAULT_CACHE_SCOPE, value, cache_duration)
+        return value
+    return default
+
+  security.declareProtected(Permissions.AccessContentsInformation, 'set')
+  def set(self, cache_id, value):
+    """
+      Set value.
+    """
+    cache_duration = self.getRamCacheFactory().cache_duration
+    ram_cache_factory_plugin_list = self.getRamCacheFactoryPluginList()
+    # set only in first plugin in sequence
+    ram_cache_factory_plugin_list[0].set(cache_id, DEFAULT_CACHE_SCOPE, value, cache_duration)
diff --git a/product/ERP5Type/Core/CacheFactory.py b/product/ERP5Type/Core/CacheFactory.py
index 56cea0ec00..3a7a2efc07 100644
--- a/product/ERP5Type/Core/CacheFactory.py
+++ b/product/ERP5Type/Core/CacheFactory.py
@@ -71,8 +71,16 @@ class CacheFactory(XMLObject):
   security.declareProtected(Permissions.AccessContentsInformation, 'getRamCacheFactory')
   def getRamCacheFactory(self):
     """ Return RAM based cache factory """
-    erp5_site_id = self.getPortalObject().getId()
-    return CachingMethod.factories[erp5_site_id][self.cache_scope]
+    cache_factory_name = self.getId()
+    cache_tool = self.portal_caches
+    cache_factory = CachingMethod.factories.get(cache_factory_name)
+    #XXX This conditional statement should be remove as soon as
+    #Broadcasting will be enable among all zeo clients.
+    #Interaction which update portal_caches should interact with all nodes.
+    if cache_factory is None and getattr(cache_tool, cache_factory_name, None) is not None:
+      #ram_cache_root is not up to date for current node
+      cache_tool.updateCache()
+    return CachingMethod.factories[cache_factory_name]
 
   security.declareProtected(Permissions.AccessContentsInformation, 'getRamCacheFactoryPluginList')
   def getRamCacheFactoryPluginList(self):
diff --git a/product/ERP5Type/Tool/CacheTool.py b/product/ERP5Type/Tool/CacheTool.py
index e3480c7cd6..83965d4386 100644
--- a/product/ERP5Type/Tool/CacheTool.py
+++ b/product/ERP5Type/Tool/CacheTool.py
@@ -67,7 +67,7 @@ class CacheTool(BaseTool):
   def getCacheFactoryList(self):
     """ Return available cache factories """
     rd = {}
-    for cf in self.objectValues('ERP5 Cache Factory'):
+    for cf in self.objectValues(['ERP5 Cache Factory', 'ERP5 Cache Bag']):
       cache_scope = cf.getId()
       rd[cache_scope] = {}
       rd[cache_scope]['cache_plugins'] = []
-- 
2.30.9