Initial import of TIDStorage product.

git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@24513 20353a03-c40f-0410-a6d1-a30d3c3de9de

product/TIDStorage/ExchangeProtocol.py

+############################################################################
+#
+# Copyright (c) 2007, 2008 Nexedi SARL and Contributors. All Rights Reserved.
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+class ClientDisconnected(Exception):
+  pass
+
+class ExchangeProtocol:
+  """
+    Handle data exchange between client and server.
+    Kinds of data which can be exchanged:
+     - str
+       send_field
+       recv_field
+     - int
+       send_int
+       recv_int
+     - list of str
+       send_list
+       recv_list
+     - list of int
+       send_int_list
+       recv_int_list
+     - dict (key: str, value: int)
+       send_dict
+       recv_dict
+    Forbidden chars:
+      Send (raise if present):
+        \\n (field separator)
+      Receive (stripped silently):
+        \\n (field separator)
+        \\r (for compatibility)
+  """
+  def __init__(self, socket):
+    self._socket = socket
+
+  def send_field(self, to_send):
+    if type(to_send) is not str:
+      raise ValueError, 'Value is not of str type: %r' % (type(to_send), )
+    if '\n' in to_send:
+      raise ValueError, '\\n is a forbidden value.'
+    self._socket.send(to_send)
+    self._socket.send('\n')
+
+  def recv_field(self):
+    received = None
+    result = []
+    append = result.append
+    while received != '\n':
+      received = self._socket.recv(1)
+      if len(received) == 0:
+        raise ClientDisconnected
+      if received != '\r':
+        append(received)
+    return ''.join(result[:-1])
+
+  def send_int(self, to_send):
+    self.send_field(str(to_send))
+
+  def recv_int(self):
+    return int(self.recv_field())
+
+  def send_list(self, to_send, send_length=True):
+    assert isinstance(to_send, (tuple, list))
+    if send_length:
+      self.send_int(len(to_send))
+    for field in to_send:
+      self.send_field(field)
+
+  def send_int_list(self, to_send, *args, **kw):
+    self.send_list([str(x) for x in to_send], *args, **kw)
+
+  def recv_list(self, length=None):
+    result = []
+    append = result.append
+    if length is None:
+      length = int(self.recv_field())
+    for field_number in xrange(length):
+      append(self.recv_field())
+    return result
+
+  def recv_int_list(self, *args, **kw):
+    return [int(x) for x in self.recv_list(*args, **kw)]
+
+  def send_dict(self, to_send):
+    """
+      Key: string
+      Value: int
+    """
+    assert isinstance(to_send, (dict))
+    if len(to_send) == 0:
+      key_list = value_list = []
+    else:
+      key_list, value_list = zip(*to_send.items())
+    self.send_list(key_list)
+    self.send_int_list(value_list, send_length=False)
+
+  def recv_dict(self):
+    """
+      Key: string
+      Value: int
+    """
+    key_list = self.recv_list()
+    value_list = self.recv_int_list(len(key_list))
+    result = dict(zip(key_list, value_list))
+    return result
+

product/TIDStorage/README

+1) Protocole:
+ Tous caractères autorisés dans les données, à l'exception de \n et \r.
+ Tout champ se termine par un \n (\r ignoré).
+ Pas d'échappement.
+ Lors de transfert de listes, la liste est précédée par le nombre de champs qu'elle contient.
+  Ex:
+  3\n
+  foo\n
+  bar\n
+  baz\n
+
+2) Commande de début de commit:
+
+BEGIN\n
+<identifiant du commit>\n
+<liste des storages concernés>
+
+ <identifiant du commit>: doit être identique à celui fourni à la fin de l'opération (que ça soit un ABORT ou un COMMIT)
+ <liste des storages concernés>: liste des identifiants des storages concernés par le commit
+  NB: la liste se termine par un \n, il n'est donc pas répété ici.
+
+Réponse: (rien)
+
+3) Commande d'annulation de la transaction:
+
+ABORT\n
+<identifiant du commit>\n
+
+  <identifiant du commit>: (cf. BEGIN)
+
+Réponse: (rien)
+
+4) Commande de finalisation de la transaction:
+
+COMMIT\n
+<identifiant du commit>\n
+<liste des storages concernés>
+<liste des TIDs commités>
+
+ <identifiant du commit>: (cf. BEGIN)
+ <liste des storages concernés>: (cf. BEGIN)
+ <liste des TIDs commités>: De même longueur que la liste des storages concernés. L'ordre doit corresponde à cette dernière.
+  NB: la liste se termine par un \n, il n'est donc pas répété ici.
+
+Réponse: (rien)
+
+5) Commande de lecture des données:
+
+DUMP\n
+
+Réponse:
+<liste des storages>
+<liste des TIDs>
+

product/TIDStorage/ZEOClientStorage.py

+############################################################################
+#
+# Copyright (c) 2007 Nexedi SARL and Contributors. All Rights Reserved.
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from ZEO.ClientStorage import ClientStorage
+
+LAST_COMMITED_TID_PROPERTY_ID = '_last_commited_tid'
+
+# Hook tpc_finish's hook method.
+# New hook must be a local method because it must access tpc_finish's "self"
+# and original hook.
+
+original_tpc_finish = ClientStorage.tpc_finish
+def tpc_finish(self, txn, f=None):
+  def saveTIDOnInstance(tid):
+    if f is not None:
+      f(tid)
+    setattr(self, LAST_COMMITED_TID_PROPERTY_ID, tid)
+  return original_tpc_finish(self, txn, f=saveTIDOnInstance)
+ClientStorage.tpc_finish = tpc_finish
+
+def getLastCommitedTID(self):
+  """
+    Return last commited tid for this storage, or None if no transaction
+    was commited yet.
+  """
+  return getattr(self, LAST_COMMITED_TID_PROPERTY_ID, None)
+ClientStorage.getLastCommitedTID = getLastCommitedTID
+

product/TIDStorage/__init__.py

+############################################################################
+#
+# Copyright (c) 2007, 2008 Nexedi SARL and Contributors. All Rights Reserved.
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+# Load monkey patches
+import transaction_transaction
+import ZEOClientStorage
+

product/TIDStorage/repozo/from_z2.8.8_repozo.diff

+--- /home/vincent/bin/zope2.8/bin/repozo.py	2007-02-09 13:52:35.000000000 +0100
++++ repozo.py	2007-10-26 15:30:43.311046075 +0200
+@@ -50,6 +50,12 @@
+         Compress with gzip the backup files.  Uses the default zlib
+         compression level.  By default, gzip compression is not used.
+ 
++    -m / --max-tid
++        Stop at given TID when saving the Data.fs.
++
++    -M / --print-max-tid
++        Print the last saved transaction's tid.
++
+ Options for -R/--recover:
+     -D str
+     --date=str
+@@ -70,6 +76,7 @@
+ import time
+ import errno
+ import getopt
++import base64
+ 
+ from ZODB.FileStorage import FileStorage
+ 
+@@ -104,10 +111,11 @@
+ def parseargs():
+     global VERBOSE
+     try:
+-        opts, args = getopt.getopt(sys.argv[1:], 'BRvhf:r:FD:o:Qz',
++        opts, args = getopt.getopt(sys.argv[1:], 'BRvhf:r:FD:o:Qzm:M',
+                                    ['backup', 'recover', 'verbose', 'help',
+                                     'file=', 'repository=', 'full', 'date=',
+-                                    'output=', 'quick', 'gzip'])
++                                    'output=', 'quick', 'gzip', 'max-tid=',
++                                    'print-max-tid'])
+     except getopt.error, msg:
+         usage(1, msg)
+ 
+@@ -120,6 +128,8 @@
+         output = None       # where to write recovered data; None = stdout
+         quick = False       # -Q flag state
+         gzip = False        # -z flag state
++        print_tid = False   # -M flag state
++        max_tid = None      # -m argument, if any
+ 
+     options = Options()
+ 
+@@ -150,6 +160,10 @@
+             options.output = arg
+         elif opt in ('-z', '--gzip'):
+             options.gzip = True
++        elif opt in ('-M', '--print-max-tid'):
++             options.print_tid = True
++        elif opt in ('-m', '--max-tid'):
++             options.max_tid = base64.decodestring(arg)
+         else:
+             assert False, (opt, arg)
+ 
+@@ -174,6 +188,12 @@
+         if options.file is not None:
+             log('--file option is ignored in recover mode')
+             options.file = None
++        if options.print_tid:
++            log('--print-max-tid is ignored in recover mode')
++            options.print_tid = False
++        if options.max_tid is not None:
++            log('--max-tid is ignored in recover mode')
++            options.max_tid = None
+     return options
+ 
+ 
+@@ -349,13 +369,19 @@
+ 
+ def do_full_backup(options):
+     # Find the file position of the last completed transaction.
+-    fs = FileStorage(options.file, read_only=True)
++    fs = FileStorage(options.file, read_only=True, stop=options.max_tid)
+     # Note that the FileStorage ctor calls read_index() which scans the file
+     # and returns "the position just after the last valid transaction record".
+     # getSize() then returns this position, which is exactly what we want,
+     # because we only want to copy stuff from the beginning of the file to the
+     # last valid transaction record.
+     pos = fs.getSize()
++    if options.print_tid:
++      undo_log = fs.undoLog(last=-1)
++      if len(undo_log):
++        print >> sys.stdout, 'Last TID: %s' % (undo_log[0]['id'], )
++      else:
++        print >> sys.stderr, 'Cannot get latest TID'
+     fs.close()
+     options.full = True
+     dest = os.path.join(options.repository, gen_filename(options))
+@@ -375,13 +401,19 @@
+ 
+ def do_incremental_backup(options, reposz, repofiles):
+     # Find the file position of the last completed transaction.
+-    fs = FileStorage(options.file, read_only=True)
++    fs = FileStorage(options.file, read_only=True, stop=options.max_tid)
+     # Note that the FileStorage ctor calls read_index() which scans the file
+     # and returns "the position just after the last valid transaction record".
+     # getSize() then returns this position, which is exactly what we want,
+     # because we only want to copy stuff from the beginning of the file to the
+     # last valid transaction record.
+     pos = fs.getSize()
++    if options.print_tid:
++      undo_log = fs.undoLog(last=-1)
++      if len(undo_log):
++        print >> sys.stdout, 'Last TID: %s' % (undo_log[0]['id'], )
++      else:
++        print >> sys.stderr, 'Cannot get latest TID'
+     fs.close()
+     options.full = False
+     dest = os.path.join(options.repository, gen_filename(options))

product/TIDStorage/repozo/restore_tidstorage.py

+#!/usr/bin/python
+
+##############################################################################
+#
+# Copyright (c) 2007 Nexedi SARL. All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+# Parts of this file are borrowed from Zope 2.8.8 repozo.py script.
+# Essentialy "usage", "parseargs" and parts of "restore" methods.
+# So it's released under the ZPL v2.0, as is Zope 2.8.8 .
+
+"""
+Usage: %(program)s [-h|--help] [-c|--config configuration_file]
+
+  -h
+  --help
+    Display this help and exit.
+
+  -c configuration_file
+  --config configuration_file
+    Use given file as configuration file.
+    It must be a python file.
+    Recquired if neither -h nor --help are given.
+"""
+
+import imp
+import getopt
+import sys
+import os
+# urllib2 does not support (?) urls containing credentials
+# (http://login:password@...) but it's fine with urllib.
+from struct import pack
+import shutil
+from ZODB.FileStorage import FileStorage
+
+program = sys.argv[0]
+
+def log(message):
+  print message
+
+def parse(status_file):
+  tid_log = open(status_file)
+  content = {}
+  last_timestamp = None
+
+  line = tid_log.readline()
+  while line != '':
+    split_line = line.split(' ', 2)
+    assert len(split_line) == 3, repr(split_line)
+    line_timestamp, line_type, line_dict = split_line
+    line_timestamp = float(line_timestamp)
+    assert line_type in ('f', 'd'), repr(line_type)
+    if last_timestamp is None:
+      last_timestamp = line_timestamp
+    else:
+      assert last_timestamp < line_timestamp, '%r < %r' % (last_timestamp, line_timestamp)
+    line_dict = eval(line_dict, None)
+    assert isinstance(line_dict, dict), type(line_dict)
+    assert len(line_dict), repr(line_dict)
+    if line_type == 'd':
+      for key, value in line_dict.iteritems():
+        if key in content:
+          assert content[key] < value, '%r < %r' % (content[key], value)
+        content[key] = value
+    elif line_type == 'f':
+      for key, value in content.iteritems():
+        assert key in line_dict, repr(key)
+        assert value <= line_dict[key], '%r <= %r' % (value, line_dict[key])
+      content = line_dict
+    line = tid_log.readline()
+  return content
+
+READCHUNK = 10 * 1024 * 1024
+
+def recover(data_fs_backup_path_dict, status_file):
+  last_tid_dict = parse(status_file)
+  for storage_id, (file_path, backup_path) in data_fs_backup_path_dict.iteritems():
+    # Derived from repozo (function=do_full_backup)
+    # TODO: optimise to read backup only once.
+    can_restore = False
+    if os.path.exists(backup_path):
+      if os.path.exists(file_path):
+        print 'Both original and backup files exist for %r. If previous restoration was successful, you should delete the backup for this restoration to take place. Original: %r Backup: %r' % (storage_id, file_path, backup_path)
+      else:
+        print 'Only backup file is available for %r: %r. Assuming it\'s ok and restoring to %r' % (storage_id, backup_path, file_path)
+        can_restore = True
+    else:
+      if os.path.exists(file_path):
+        sys.stdout.write('Copying %r to %r... ' % (file_path, backup_path))
+        shutil.copy(file_path, backup_path)
+        initial_size = stat(file_path).st_size
+        final_size = stat(backup_path).st_size
+        if initial_size == final_size:
+          can_restore = True
+          print 'Done.'
+        else:
+          print 'Backup size %i differs from original size %i. Is the original file (%r) still in use ? Is there enough free disk space at destination (%r) ?' % (final_size, initial_size, file_path, backup_path)
+      else:
+        print 'Cannot find any file for %r: %r and %r do not exist.' % (storage_id, file_path, backup_path)
+    if can_restore:
+      last_tid = last_tid_dict[storage_id] + 1
+      tid = pack('>Q', last_tid)
+      # Find the file position of the last completed transaction.
+      fs = FileStorage(backup_path, read_only=True, stop=tid)
+      # Note that the FileStorage ctor calls read_index() which scans the file
+      # and returns "the position just after the last valid transaction record".
+      # getSize() then returns this position, which is exactly what we want,
+      # because we only want to copy stuff from the beginning of the file to the
+      # last valid transaction record.
+      pos = fs.getSize()
+      fs.close()
+      print 'Restoring backup: %s bytes (transaction %r) from %s to %s' % (pos, tid, backup_path, file_path)
+      source_file = open(backup_path, 'rb')
+      destination_file = open(file_path, 'wb')
+      while pos:
+        todo = min(READCHUNK, pos)
+        data = source_file.read(todo)
+        if not data:
+          print 'Unexpected end of data stream (should contain %i more bytes)' % (pos, )
+          break
+        destination_file.write(data)
+        pos -= len(data)
+      destination_file.close()
+      source_file.close()
+    else:
+      print 'Skipping restoration of %r (%r).' % (file_path, storage_id)
+
+def usage(code, msg=''):
+  outfp = sys.stderr
+  if code == 0:
+    outfp = sys.stdout
+
+  print >> outfp, __doc__ % globals()
+  if msg:
+    print >> outfp, msg
+
+  sys.exit(code)
+
+def parseargs():
+  try:
+    opts, args = getopt.getopt(sys.argv[1:], 'hc:',
+                               ['help', 'config='])
+  except getopt.error, msg:
+    usage(1, msg)
+
+  class Options:
+    configuration_file_name = None
+    status_file = None
+
+  options = Options()
+
+  for opt, arg in opts:
+    if opt in ('-h', '--help'):
+      usage(0)
+    elif opt in ('-c', '--config'):
+      options.configuration_file_name = arg
+
+  if options.configuration_file_name is None:
+    usage(1, 'Either -c or --config is required.')
+
+  configuration_filename, ext = os.path.splitext(os.path.basename(options.configuration_file_name))
+  configuration_path = os.path.dirname(options.configuration_file_name)
+  if len(configuration_path):
+    configuration_path = [configuration_path]
+  else:
+    configuration_path = sys.path
+  file, path, description = imp.find_module(configuration_filename, configuration_path)
+  module = imp.load_module(configuration_filename, file, path, description)
+  file.close()
+  try:
+    options.data_fs_backup_path_dict = module.data_fs_backup_path_dict
+    options.status_file = module.status_file
+  except AttributeError, msg:
+    usage(1, msg)
+  return options
+
+options = parseargs()
+recover(
+  data_fs_backup_path_dict=options.data_fs_backup_path_dict,
+  status_file=options.status_file)
+

product/TIDStorage/repozo/sample_configuration.py

+# COMMON
+# This part is used both by server_v2.py and repozo_tidstorage_v2.py
+known_tid_storage_identifier_dict = {
+  "((('localhost', 8200),), '2')":
+    ('/home/vincent/zeo2/var2/Data.fs',
+     '/home/vincent/tmp/repozo/z22',
+     'foo_test'),
+  "((('localhost', 8200),), '1')":
+    ('/home/vincent/zeo2/var/Data.fs',
+     '/home/vincent/tmp/repozo/z21',
+     'bar_test'),
+  "((('localhost', 8100),), '1')":
+    ('/home/vincent/zeo1/var/Data.fs',
+     '/home/vincent/tmp/repozo/z11',
+     'baz_test'),
+}
+base_url = 'http://localhost:5080/erp5/%s/modifyContext'
+port = 9001
+host = '127.0.0.1'
+
+# SERVER
+# This part is only used by server_v2.py
+#logfile_name = 'tidstorage.log'
+#pidfile_name = 'tidstorage.pid'
+#fork = False
+#setuid = None
+#setgid = None
+status_file = 'tidstorage.tid'
+burst_period = 30
+full_dump_period = 300
+
+# REPOZO_TIDSTORAGE
+# This part is only used by repozo_tidstorage_v2.py
+timestamp_file_path = 'repozo_tidstorage_timestamp.log'
+

product/TIDStorage/transaction_transaction.py

+############################################################################
+#
+# Copyright (c) 2007, 2008 Nexedi SARL and Contributors. All Rights Reserved.
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from ExchangeProtocol import ExchangeProtocol
+from transaction._transaction import Transaction
+from zLOG import LOG, WARNING
+import socket
+import thread
+import struct
+import sys
+
+GET_LAST_COMMITED_TID_METHOD_ID = 'getLastCommitedTID'
+TID_STORAGE_ADDRESS = ('127.0.0.1', 9001)
+
+tid_storage = None
+zope_identifier = None
+
+# Borrowed from CMFActivity.ActivityTool.getCurrentNode
+def getZopeId():
+  """ Return current node in form ip:port """
+  global zope_identifier
+  if zope_identifier is None:
+    port = ''
+    from asyncore import socket_map
+    for k, v in socket_map.items():
+      if hasattr(v, 'port'):
+        # see Zope/lib/python/App/ApplicationManager.py: def getServers(self)
+        type = str(getattr(v, '__class__', 'unknown'))
+        if type == 'ZServer.HTTPServer.zhttp_server':
+          port = v.port
+          break
+    assert port != '', 'zhttp_server not started yet'
+    ip = socket.gethostbyname(socket.gethostname())
+    if TID_STORAGE_ADDRESS[0] != '127.0.0.1':
+      assert ip != '127.0.0.1', 'self address must not be 127.0.0.1 if TIDStorage is remote'
+    zope_identifier = '%s:%s' %(ip, port)
+  return zope_identifier
+
+def getFilestorageList(resource_list):
+  return getFilestorageToTIDMapping(resource_list).keys()
+
+def getFilestorageToTIDMapping(resource_list):
+  datafs_tid_update_dict = {}
+  for resource in resource_list:
+    storage = getattr(resource, '_storage', None)
+    if storage is not None:
+      getLastCommitedTID = getattr(storage, GET_LAST_COMMITED_TID_METHOD_ID,
+                                   None)
+      if getLastCommitedTID is not None:
+        tid = getLastCommitedTID()
+        _addr = tuple([tuple(x) for x in getattr(storage, '_addr', [])])
+        _storage = getattr(storage, '_storage', '')
+        datafs_id = repr((_addr, _storage))
+        assert datafs_id not in datafs_tid_update_dict
+        if tid is None:
+          datafs_tid_update_dict[datafs_id] = None
+        else:
+          # unpack stolen from ZODB/utils.py:u64
+          datafs_tid_update_dict[datafs_id] = struct.unpack(">Q", tid)[0]
+  return datafs_tid_update_dict
+
+class BufferedSocket:
+  """
+    Write-only thread-safe buffered socket.
+    Attemps to reconnect at most once per flush.
+  """
+
+  _socket_lock = thread.allocate_lock()
+  _connected = False
+  
+  def __init__(self, address):
+    self._socket = socket.socket()
+    self._address = address
+    self._send_buffer_dict = {}
+
+  def _connect(self):
+    try:
+      self._socket.connect(self._address)
+      self._notifyConnected()
+    except socket.error, message:
+      # We don't want to have an error line per failed connection attemp, to
+      # avoid flooding the logfile.
+      pass
+
+  def _getSendBuffer(self, ident):
+    send_buffer = self._send_buffer_dict.get(ident)
+    if send_buffer is None:
+      send_buffer = self._send_buffer_dict[ident] = []
+    return send_buffer
+
+  def _notifyDisconnected(self, message):
+    if self._connected:
+      self._connected = False
+      LOG('TIDStorage', WARNING, 'Disconnected: %s' % (message, ))
+
+  def _notifyConnected(self):
+    if not self._connected:
+      self._connected = True
+      # Display a log message at WARNING level, so that reconnection message
+      # are visible when disconnection messages are visible, even if it is
+      # not a warning, properly speaking.
+      LOG('TIDStorage', WARNING, 'Connected')
+
+  def send(self, to_send):
+    send_buffer = self._getSendBuffer(thread.get_ident())
+    send_buffer.append(to_send)
+
+  def flush(self):
+    """
+      Flush send buffer and actually send data, with extra checks to behave
+      nicely if connection is broken.
+      Do not retry to send if something goes wrong (data is then lost !).
+      Here, most important thing is speed, not data.
+      Serialize usage.
+    """
+    ident = thread.get_ident()
+    self._socket_lock.acquire()
+    try:
+      if not self._connected:
+        self._connect()
+      if self._connected:
+        try:
+          self._socket.sendall(''.join(self._getSendBuffer(ident)))
+        except socket.error, message:
+          self._notifyDisconnected(message)
+          try:
+            self._socket.shutdown(socket.SHUT_RDWR)
+          except socket.error:
+            self._socket.close()
+          self._socket = socket.socket()
+    finally:
+      self._socket_lock.release()
+    self._send_buffer_dict[ident] = []
+
+class TIDClient:
+  def __init__(self, address):
+    self._buffered_socket = BufferedSocket(address)
+    self._field_exchange = ExchangeProtocol(socket=self._buffered_socket)
+
+  def commit(self, tid_update_dict):
+    """
+      Send given dict to TIDStorage server.
+    """
+    self._send_command('commit')
+    self._field_exchange.send_dict(tid_update_dict)
+    self._buffered_socket.flush()
+
+  def begin(self, storage_id_list):
+    """
+      Inform TIDStorage connection tracking that commit was initiated.
+    """
+    self._send_command('begin')
+    self._field_exchange.send_list(storage_id_list)
+    self._buffered_socket.flush()
+  
+  def abort(self):
+    """
+      Inform TIDStorage connection tracking that commit was aborted.
+    """
+    self._send_command('abort')
+    self._buffered_socket.flush()
+
+  def _send_command(self, command):
+    """
+      Every command must be followed by an identifier.
+      This identifier is used to track transactions, so the same identifier
+      must not be used twice at the same time, but can be reused later.
+    """
+    self._field_exchange.send_field(command)
+    self._field_exchange.send_field('%s_%x' % (getZopeId(), thread.get_ident()))
+
+original__commitResources = Transaction._commitResources
+def _commitResources(self, *args, **kw):
+  """
+    Hook Transaction's _commitResources.
+
+    Before:
+     - Initialise TIDClient if needed
+     - Check if there is any storage we are interested in in current commit
+     - If so, issue a begin
+    
+    After (2 cases):
+     - original__commitResources raised:
+       - Issue an abort
+     - otherwise:
+       - Issue a commit
+
+    Note to editors: Prevent your code from raising anything ! This method
+    MUST NOT raise any exception, except that it MUST NOT hide any exception
+    raised by original__commitResources.
+    """
+  has_storages = False
+  try:
+    global tid_storage
+    if tid_storage is None:
+      tid_storage = TIDClient(TID_STORAGE_ADDRESS)
+    filestorage_list = getFilestorageList(self._resources)
+    if len(filestorage_list):
+      has_storages = True
+      tid_storage.begin(filestorage_list)
+  except:
+    LOG('TIDStorage _commitResources', WARNING, 'Exception in begin phase', error=sys.exc_info())
+  try:
+    result = original__commitResources(self, *args, **kw)
+  except:
+    if has_storages:
+      exception = sys.exc_info()
+      try:
+        tid_storage.abort()
+      except:
+        LOG('TIDStorage _commitResources', WARNING, 'Exception in abort phase', error=sys.exc_info())
+      # Re-raise original exception, in case sendTIDCommitAbort tainted
+      # last exception value.
+      raise exception[0], exception[1], exception[2]
+    else:
+      raise
+  else:
+    if has_storages:
+      # Now that everything has been commited, all exceptions relative to added
+      # code must be swalowed (but still reported) to avoid confusing transaction
+      # system.
+      try:
+        tid_storage.commit(getFilestorageToTIDMapping(self._resources))
+      except:
+        LOG('TIDStorage _commitResources', WARNING, 'Exception in commit phase', error=sys.exc_info())
+  return result
+ 
+Transaction._commitResources = _commitResources
+

product/TIDStorage/utils/check_tid_log.py

+#!/usr/bin/python
+##############################################################################
+#
+# Copyright (c) 2008 Nexedi SARL and Contributors. All Rights Reserved.
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+# Checks the sanity of a tidstorage TID log provided via stdin.
+# Exit status:
+#  0 Success
+#  1 Failure
+# Error is displayed on stderr.
+# On success, final tid values for each storage is displayed on stdout.
+
+import sys
+content = {}
+last_timestamp = None
+
+line = sys.stdin.readline()
+while line != '':
+  split_line = line.split(' ', 2)
+  assert len(split_line) == 3, repr(split_line)
+  line_timestamp, line_type, line_dict = split_line
+  line_timestamp = float(line_timestamp)
+  assert line_type in ('f', 'd'), repr(line_type)
+  if last_timestamp is None:
+    last_timestamp = line_timestamp
+  else:
+    assert last_timestamp < line_timestamp, '%r < %r' % (last_timestamp, line_timestamp)
+  line_dict = eval(line_dict, None)
+  assert isinstance(line_dict, dict), type(line_dict)
+  assert len(line_dict), repr(line_dict)
+  if line_type == 'd':
+    for key, value in line_dict.iteritems():
+      if key in content:
+        assert content[key] < value, '%r < %r' % (content[key], value)
+      content[key] = value
+  elif line_type == 'f':
+    for key, value in content.iteritems():
+      assert key in line_dict, repr(key)
+      assert value <= line_dict[key], '%r <= %r' % (value, line_dict[key])
+    content = line_dict
+  line = sys.stdin.readline()
+
+key_list = content.keys()
+key_list.sort()
+for key in key_list:
+  print '%r %r' % (key, content[key])
+

product/TIDStorage/utils/tid_log_to_full.py

+#!/usr/bin/python
+##############################################################################
+#
+# Copyright (c) 2008 Nexedi SARL and Contributors. All Rights Reserved.
+#                    Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+# Transforms a TIDStorage TID log provided on stdin, which might contain
+# full statuses and/or incremental changes, and provides a version on stdout
+# containing only full statuses.
+# Also, does sanity checks on the given file.
+# Exit status:
+#  0 Success
+#  1 Failure
+
+import sys
+content = {}
+last_timestamp = None
+
+line = sys.stdin.readline()
+while line != '':
+  split_line = line.split(' ', 2)
+  assert len(split_line) == 3, repr(split_line)
+  line_timestamp, line_type, line_dict = split_line
+  line_timestamp = float(line_timestamp)
+  assert line_type in ('f', 'd'), repr(line_type)
+  if last_timestamp is None:
+    last_timestamp = line_timestamp
+  else:
+    assert last_timestamp < line_timestamp, '%r < %r' % (last_timestamp, line_timestamp)
+  line_dict = eval(line_dict, None)
+  assert isinstance(line_dict, dict), type(line_dict)
+  assert len(line_dict), repr(line_dict)
+  if line_type == 'd':
+    for key, value in line_dict.iteritems():
+      if key in content:
+        assert content[key] < value, '%r < %r' % (content[key], value)
+      content[key] = value
+    print '%r f %r' % (line_timestamp, content)
+  elif line_type == 'f':
+    for key, value in content.iteritems():
+      assert key in line_dict, repr(key)
+      assert value <= line_dict[key], '%r <= %r' % (value, line_dict[key])
+    content = line_dict
+    print line.strip()
+  line = sys.stdin.readline()
+