Add MemcachedTool.

This tools offers a dictionnary interface to memcached.
As it depends on python-memcached and should not fail silently if the site administrator does not have this python module, this tool is disabled by default.
You can enable it runing "touch Products/ERP5Type/USE_MEMCACHED_TOOL"


git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@11618 20353a03-c40f-0410-a6d1-a30d3c3de9de

product/ERP5Type/Tool/MemcachedTool.py

+##############################################################################
+#
+# Copyright (c) 2006 Nexedi SARL and Contributors. All Rights Reserved.
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from Products.ERP5Type import allowMemcachedTool
+from Products.ERP5Type.Tool.BaseTool import BaseTool
+from Products.ERP5Type import Permissions, _dtmldir
+from AccessControl import ClassSecurityInfo
+from Globals import DTMLFile
+
+MEMCACHED_TOOL_MODIFIED_FLAG_PROPERTY_ID = '_v_memcached_edited'
+
+if allowMemcachedTool():
+  import memcache
+  from Shared.DC.ZRDB.TM import TM
+  from Products.PythonScripts.Utility import allow_class
+  
+  MARKER = tuple()
+  UPDATE_ACTION = 'update'
+  DELETE_ACTION = 'delete'
+  
+  class MemcachedDict(TM):
+    """
+      Present memcached similarly to a dictionnary (not all method are
+      available).
+      Uses transactions to only update memcached at commit time.
+      No conflict generation/resolution : last edit wins.
+  
+      TODO:
+        - prove that concurency handling in event queuing is not needed
+        - make picklable ?
+    """
+  
+    def __init__(self, server='127.0.0.1:11211'):
+      """
+        Initialise properties :
+        memcached_connection
+          Connection to memcached.
+        local_cache
+          Dictionnary used as a connection cache with duration limited to
+          transaction length.
+        scheduled_action_dict
+          Each key in this dictionnary must be handled at transaction commit.
+          Value gives the action to take :
+            UPDATE_ACTION 
+              Take value from local cache and send it to memcached.
+            DELETE_ACTION
+              Send a delete order to memcached.
+      """
+      self.local_cache = {}
+      self.scheduled_action_dict = {}
+      self.memcached_connection = memcache.Client((server, ))
+  
+    def __del__(self):
+      """
+        Close connection before deleting object.
+      """
+      self.memcached_connection.disconnect_all()
+  
+    def _finish(self, *ignored):
+      """
+        Actually modifies the values in memcached.
+        This avoids multiple accesses to memcached during the transaction.
+        Invalidate all local cache to make sure changes donc by other zopes
+        would not be ignored.
+      """
+      for key, value in self.local_cache.iteritems():
+        if getattr(value, MEMCACHED_TOOL_MODIFIED_FLAG_PROPERTY_ID, None):
+          delattr(value, MEMCACHED_TOOL_MODIFIED_FLAG_PROPERTY_ID)
+          self.scheduled_action_dict[key] = UPDATE_ACTION
+      for key, action in self.scheduled_action_dict.iteritems():
+        if action is UPDATE_ACTION:
+          self.memcached_connection.set(key, self.local_cache[key], 0)
+        elif action is DELETE_ACTION:
+          self.memcached_connection.delete(key, 0)
+      self.scheduled_action_dict.clear()
+      self.local_cache.clear()
+  
+    def _abort(self, *ignored):
+      """
+        Cleanup the action dict and invalidate local cache.
+      """
+      self.local_cache.clear()
+      self.scheduled_action_dict.clear()
+  
+    def __getitem__(self, key):
+      """
+        Get an item from local cache, otherwise from memcached.
+        Note that because of memcached python client limitations, it never
+        raises KeyError.
+      """
+      # We need to register in this function too to be able to flush cache at 
+      # transaction end.
+      self._register()
+      result = self.local_cache.get(key, MARKER)
+      if result is MARKER:
+        result = self.memcached_connection.get(key)
+        self.local_cache[key] = result
+      return result
+  
+    def __setitem__(self, key, value):
+      """
+        Set an item to local cache and schedule update of memcached.
+      """
+      self._register()
+      self.scheduled_action_dict[key] = UPDATE_ACTION
+      self.local_cache[key] = value
+  
+    def __delitem__(self, key):
+      """
+        Delete an item from local cache and schedule deletion in memcached.
+        Never raises KeyError because action is delayed.
+      """
+      self._register()
+      self.scheduled_action_dict[key] = DELETE_ACTION
+      if key in self.local_cache:
+        del self.local_cache[key]
+  
+    def set(self, key, val, time=0):
+      """
+        Set an item to local cache and schedule update of memcached.
+      """
+      return self.__setitem__(key, value)
+  
+    def get(self, key, default=MARKER):
+      """
+        Get an item from local cache, otherwise from memcached.
+        Note that because __getitem__ never raises error, 'default' will never
+        be used (None will be returned instead).
+      """
+      return self.__getitem__(key)
+  
+  class SharedDict:
+    """
+      Class to make possible for multiple "users" to store data in the same
+      dictionnary without risking to overwrite other's data.
+      Each "user" of the dictionnary must get an instance of this class.
+  
+      TODO:
+        - handle persistence ?
+    """
+  
+    def __init__(self, dictionary, prefix):
+      """
+        dictionary
+          Instance of dictionnary to share.
+        prefix
+          Prefix used by the "user" owning an instance of this class.
+      """
+      self._v_dictionary = dictionary
+      self.prefix = prefix
+  
+    def _prefixKey(self, key):
+      """
+        Prefix key with self.prefix .
+      """
+      return '%s_%s' % (self.prefix, key)
+  
+    def __getitem__(self, key):
+      """
+        Get item from memcached.
+      """
+      return self._v_dictionary.__getitem__(self._prefixKey(key))
+    
+    def __setitem__(self, key, value):
+      """
+        Put item in memcached.
+      """
+      self._v_dictionary.__setitem__(self._prefixKey(key), value)
+  
+    def __delitem__(self, key):
+      """
+        Delete item from memcached.
+      """
+      self._v_dictionary.__delitem__(self._prefixKey(key))
+  
+    # These are the method names called by zope
+    __guarded_setitem__ = __setitem__
+    __guarded_getitem__ = __getitem__
+    __guarded_delitem__ = __delitem__
+  
+    def get(self, key, default=MARKER):
+      """
+        Get item from memcached.
+      """
+      return self.__getitem__(key)
+  
+    def set(self, key, value):
+      """
+        Put item in memcached.
+      """
+      self.__setitem__(key, value)
+  
+  allow_class(SharedDict)
+  
+  class MemcachedTool(BaseTool):
+    """
+      Memcached interface available as a tool.
+    """
+    id = "portal_memcached"
+    meta_type = "ERP5 Memcached Tool"
+    portal_type = "Memcached Tool"
+    
+    security = ClassSecurityInfo()
+    manage_options = ({'label': 'Configure',
+                       'action': 'memcached_tool_configure',
+                      },) + BaseTool.manage_options
+  
+    memcached_tool_configure = DTMLFile('memcached_tool_configure', _dtmldir)
+  
+    def _getMemcachedDict(self):
+      """
+        Return used memcached dict.
+        Create it if does not exist.
+      """
+      dictionary = getattr(self, '_v_memcached_dict', None)
+      if dictionary is None:
+        dictionary = MemcachedDict(self.getServerAddress())
+        setattr(self, '_v_memcached_dict', dictionary)
+      return dictionary
+  
+    security.declareProtected(Permissions.AccessContentsInformation, 'getMemcacheDict')
+    def getMemcachedDict(self, key_prefix):
+      """
+        Returns an object which can be used as a dict and which gets from/stores
+        to memcached server.
+        
+        key_prefix
+          Mendatory argument allowing different tool users from sharing the same
+          dictionnary key namespace.
+      """
+      return SharedDict(dictionary=self._getMemcachedDict(), prefix=key_prefix)
+  
+    security.declareProtected(Permissions.ModifyPortalContent, 'setServerAddress')
+    def setServerAddress(self, value):
+      """
+        Upon server address change, force next access to memcached dict to
+        reconnect to new ip.
+  
+        This is safe in multi zope environment, since we modify self which is
+        persistent. Then all zopes will have to reload this tool instance, and
+        loose their volatile properties in the process, which will force them
+        to reconnect to memcached.
+      """
+      self.server_address = value
+      setattr(self, '_v_memcached_dict', None)
+  
+    security.declareProtected(Permissions.AccessContentsInformation, 'getServerAddress')
+    def getServerAddress(self):
+      """
+        Return server address.
+        Defaults to 127.0.0.1:11211 if not set.
+      """
+      return getattr(self, 'server_address', '127.0.0.1:11211')
+
+else:
+  
+  class MemcachedTool(BaseTool):
+    """
+      Dummy MemcachedTool placeholder.
+    """
+    id = "portal_memcached"
+    meta_type = "ERP5 Memcached Tool"
+    portal_type = "Memcached Tool"
+
+    security = ClassSecurityInfo()
+    manage_options = ({'label': 'Configure',
+                       'action': 'memcached_tool_configure',
+                      },) + BaseTool.manage_options
+
+    def getMemcachedDict(self, *args, **kw):
+      """
+        if this function is called and memcachedtool is disabled, fail loudly
+        with a meaningfull message.
+      """
+      from Products.ERP5Type import memcached_tool_enable_path
+      raise NotImplementedError, 'MemcachedTool is disabled. You should ask the'\
+        'server administrator to enable it by creating the file %s' % (memcached_tool_enable_path, )
+
+    memcached_tool_configure = getMemcachedDict = setServerAddress = getMemcachedDict
+

product/ERP5Type/__init__.py

@@ -50,13 +50,17 @@ import Products.Localizer # So that we make sure Globals.get_request is availabl
 # read permissions for zope - this prevents security holes in 
 # production environment  
 class_tool_security_path = '%s%s%s' % (product_path, os.sep, 'ALLOW_CLASS_TOOL')
+memcached_tool_enable_path = '%s%s%s' % (product_path, os.sep, 'USE_MEMCACHED_TOOL')
 
 def allowClassTool():
   return os.access(class_tool_security_path, os.F_OK)
-  
+
+def allowMemcachedTool():
+  return os.access(memcached_tool_enable_path, os.F_OK)
+
 def initialize( context ):
   # Import Product Components
-  from Tool import ClassTool, CacheTool
+  from Tool import ClassTool, CacheTool, MemcachedTool
   import Document
   import Base, XMLObject
   from ERP5Type import ERP5TypeInformation
@@ -65,7 +69,8 @@ def initialize( context ):
   content_constructors = ()
   content_classes = ( Base.Base, XMLObject.XMLObject, )
   portal_tools = ( ClassTool.ClassTool,
-                   CacheTool.CacheTool, )
+                   CacheTool.CacheTool,
+                   MemcachedTool.MemcachedTool, )
   # Do initialization step
   initializeProduct(context, this_module, globals(),
                          document_module = Document,

product/ERP5Type/dtml/memcached_tool_configure.dtml

+<dtml-var manage_page_header>
+<dtml-var manage_tabs>
+
+<h3>Memcached server configuration</h3>
+<p>
+Memcached Tool provides the programer with an interface to a memcached server.
+This interface is similar to a dictionnary.
+The server to connet to can be configured below.
+Programer can get an interface in his scripts using :
+</p>
+<pre>portal_memcached.getMemcachedDict(key_prefix=something)</pre>
+<p>
+<i>something</i> : Must be a string uniquely identifying the usage of memcached, similar to <i>CachingMethod</i>'s id parameter. This allows multiple memcached users to share the same connection to the same memcached server without overwriting other's values.
+</p>
+<form action="setServerAddress" method="POST">
+  <input type="text" name="value" value="<dtml-var expr="context.getServerAddress()">">
+  <br/>
+  <input type="submit" value="Update server address"/>
+</form>
+
+<dtml-var manage_page_footer>