SQLNonContinuousIncreasingIdGenerator.py 11.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33
##############################################################################
#
# Copyright (c) 2010 Nexedi SARL and Contributors. All Rights Reserved.
#                    Daniele Vanbaelinghem <daniele@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################

import zope.interface
from Acquisition import aq_base
from AccessControl import ClassSecurityInfo
from Products.ERP5Type.Globals import PersistentMapping
from Products.ERP5Type import Permissions, PropertySheet, interfaces
34
from Products.ERP5Type.Utils import ScalarMaxConflictResolver
35 36
from Products.ERP5.Document.IdGenerator import IdGenerator
from _mysql_exceptions import ProgrammingError
37
from MySQLdb.constants.ER import NO_SUCH_TABLE
38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
from zLOG import LOG, INFO

class SQLNonContinuousIncreasingIdGenerator(IdGenerator):
  """
    Generate some ids with mysql storage and also zodb is enabled
    by the checkbox : StoredInZodb
  """
  zope.interface.implements(interfaces.IIdGenerator)
  # CMF Type Definition
  meta_type = 'ERP5 SQL Non Continous Increasing Id Generator'
  portal_type = 'SQL Non Continous Increasing Id Generator'
  add_permission = Permissions.AddPortalContent

  # Declarative security
  security = ClassSecurityInfo()
  security.declareObjectProtected(Permissions.AccessContentsInformation)

  # Declarative property
  property_sheets = (PropertySheet.SQLIdGenerator,
                    ) + IdGenerator.property_sheets

59 60
  last_max_id_dict = None

61 62 63 64
  def _generateNewId(self, id_group, id_count=1, default=None):
    """
      Return the next_id with the last_id with the sql method
      Store the last id on a database in the portal_ids table
65 66 67
      If stored in zodb is enable, to store the last id use
      ScalarMaxConflictResolver inspired by BTrees.Length to manage
      conflict in the zodb, use also a persistant
68 69 70 71 72 73 74 75 76 77
      mapping to be persistent
    """
    # Check the arguments
    if id_group in (None, 'None'):
      raise ValueError, '%s is not a valid group Id.' % (repr(id_group), )
    if default is None:
      default = 0

    # Retrieve the zsql method
    portal = self.getPortalObject()
78 79 80
    result_query = portal.IdTool_zGenerateId(id_group=id_group,
                                             id_count=id_count,
                                             default=default)
81 82 83 84
    try:
      # Tries of generate the new_id
      new_id = result_query[0]['LAST_INSERT_ID()']
      # Commit the changement of new_id
85
      portal.IdTool_zCommit()
86 87 88
    except ProgrammingError, error:
      if error[0] != NO_SUCH_TABLE:
        raise
89
      # If the database not exist, initialize the generator
90 91 92
      self.initializeGenerator()
    if self.getStoredInZodb():
      # Store the new_id on ZODB if the checkbox storedInZodb is enabled
93
      last_max_id_dict = self.last_max_id_dict
94
      if last_max_id_dict is None:
95 96
        # If the dictionary not exist, initialize the generator
        self.initializeGenerator()
97
        last_max_id_dict = self.last_max_id_dict
98 99 100
      if last_max_id_dict.get(id_group, None) is not None and \
          last_max_id_dict[id_group].value > new_id:
        raise ValueError, 'The last_id %s stored in zodb dictionary is ' \
101 102 103
            'higher than the new id %s generated for id_group %s. ' \
            'invoke %s/rebuildSqlTable to fix this problem.' % \
            (last_max_id_dict[id_group].value, new_id, id_group, self.absolute_url())
104 105 106 107 108
      # Check the store interval to store the data
      store_interval = self.getStoreInterval()
      if not store_interval:
        store_interval = 1
      # Store the new id
109 110
      if last_max_id_dict.get(id_group, None) is None:
        last_max_id_dict[id_group] = ScalarMaxConflictResolver(new_id)
111 112
      elif last_max_id_dict[id_group].value <= (new_id - store_interval):
        last_max_id_dict[id_group].set(new_id)
113 114
    return new_id

115 116 117 118 119
  def _updateSqlTable(self):
    """
      Update the portal ids table with the data of persistent dictionary
    """
    portal = self.getPortalObject()
120 121
    get_value_list = portal.IdTool_zGetValueList
    set_last_id_method = portal.IdTool_zSetLastId
122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
    id_group_done = []
    # Save the last id of persistent dict if it is higher that
    # the last id stored in the sql table
    for line in get_value_list().dictionaries():
      id_group = line['id_group']
      last_id = line['last_id']
      if self.last_max_id_dict.has_key(id_group) and \
        self.last_max_id_dict[id_group].value > last_id:
        set_last_id_method(id_group=id_group,
            last_id=self.last_max_id_dict[id_group].value)
      id_group_done.append(id_group)
   
    # save the last ids which not exist in sql
    for id_group in (set(self.last_max_id_dict) - set(id_group_done)):
      set_last_id_method(id_group=id_group,
          last_id=self.last_max_id_dict[id_group].value)

139 140 141 142 143 144
  security.declareProtected(Permissions.AccessContentsInformation,
      'generateNewId')
  def generateNewId(self, id_group=None, default=None):
    """
      Generate the next id in the sequence of ids of a particular group
    """
145
    return self._generateNewId(id_group=id_group, default=default)
146 147 148 149 150 151 152

  security.declareProtected(Permissions.AccessContentsInformation,
      'generateNewIdList')
  def generateNewIdList(self, id_group=None, id_count=1, default=None):
    """
      Generate a list of next ids in the sequence of ids of a particular group
    """
153 154 155
    new_id = 1 + self._generateNewId(id_group=id_group, id_count=id_count,
                                     default=default)
    return range(new_id - id_count, new_id)
156 157 158 159 160 161 162 163 164 165 166

  security.declareProtected(Permissions.AccessContentsInformation,
      'initializeGenerator')
  def initializeGenerator(self):
    """
      Initialize generator. This is mostly used when a new ERP5 site
      is created. Some generators will need to do some initialization like
      prepare some data in ZODB
    """
    LOG('initialize SQL Generator', INFO, 'Id Generator: %s' % (self,))
    # Check the dictionnary
167
    if self.last_max_id_dict is None:
168 169 170 171
      self.last_max_id_dict = PersistentMapping()
    # Create table portal_ids if not exists
    portal = self.getPortalObject()
    try:
172
      portal.IdTool_zGetValueList()
173 174 175
    except ProgrammingError, error:
      if error[0] != NO_SUCH_TABLE:
        raise
176 177
      portal.IdTool_zDropTable()
      portal.IdTool_zCreateEmptyTable()
178 179 180

    # XXX compatiblity code below, dump the old dictionnaries
    # Retrieve the zsql_method
181 182 183
    portal_ids = portal.portal_ids
    get_last_id_method = portal.IdTool_zGetLastId
    set_last_id_method = portal.IdTool_zSetLastId
184 185
    storage = self.getStoredInZodb()
    # Recovery last_max_id_dict datas in zodb if enabled and is in mysql
186 187
    if len(self.last_max_id_dict) == 0 and \
      getattr(portal_ids, 'dict_length_ids', None) is not None:
188 189 190 191 192 193 194 195
      dump_dict = portal_ids.dict_length_ids
      for id_group, last_id in dump_dict.items():
        last_insert_id = get_last_id_method(id_group=id_group)
        if len(last_insert_id) != 0:
          last_insert_id = last_insert_id[0]['LAST_INSERT_ID()']
          if last_insert_id > last_id.value:
            # Check value in dict
            if storage and (not self.last_max_id_dict.has_key(id_group) or \
196
                self.last_max_id_dict[id_group].value < last_insert_id):
197
              self.last_max_id_dict[id_group] = ScalarMaxConflictResolver(last_insert_id)
198 199 200 201
            continue
        last_id = int(last_id.value)
        set_last_id_method(id_group=id_group, last_id=last_id)
        if storage:
202
          self.last_max_id_dict[id_group] = ScalarMaxConflictResolver(last_id)
203 204

    # Store last_max_id_dict in mysql
205
    if storage:
206
      self._updateSqlTable()
207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223

  security.declareProtected(Permissions.AccessContentsInformation,
      'clearGenerator')
  def clearGenerator(self):
    """
      Clear generators data. This can be usefull when working on a
      development instance or in some other rare cases. This will
      loose data and must be use with caution

      This can be incompatible with some particular generator implementation,
      in this case a particular error will be raised (to be determined and
      added here)
    """
    # Remove dictionary
    self.last_max_id_dict = PersistentMapping()
    # Remove and recreate portal_ids table
    portal = self.getPortalObject()
224 225
    portal.IdTool_zDropTable()
    portal.IdTool_zCreateEmptyTable()
226

227 228 229 230 231 232 233 234 235 236 237
  security.declareProtected(Permissions.ModifyPortalContent,
      'exportGeneratorIdDict')
  def exportGeneratorIdDict(self):
    """
      Export last id values in a dictionnary in the form { group_id : last_id }
    """
    portal = self.getPortalObject()
    # Store last_max_id_dict in mysql
    if self.getStoredInZodb(): 
      self._updateSqlTable()
    # Return values from sql 
238
    get_value_list = portal.IdTool_zGetValueList
239 240 241 242 243 244 245 246 247 248 249 250 251
    return dict([(line['id_group'],int(line['last_id'])) for line in
      get_value_list().dictionaries()])

  security.declareProtected(Permissions.ModifyPortalContent,
      'importGeneratorIdDict')
  def importGeneratorIdDict(self, id_dict=None, clear=False):
    """
      Import data, this is usefull if we want to replace a generator by
      another one.
    """
    if clear:
      self.clearGenerator()
    portal = self.getPortalObject()
252
    set_last_id_method = portal.IdTool_zSetLastId
253 254 255 256 257 258 259 260 261 262 263 264
    if not isinstance(id_dict, dict):
      raise TypeError, 'the argument given is not a dictionary'
    new_id_dict = dict()
    for key, value in id_dict.items():
      if isinstance(value, int):
        set_last_id_method(id_group=key, last_id=value)
        # The id must be a ScalarMaxConflictResolver object for the persistent dict
        new_id_dict[key] = ScalarMaxConflictResolver(value)
      else:
        raise TypeError, 'the value in the dictionary given is not a integer'
    # Update persistent dict
    if self.getStoredInZodb():
265
      if self.last_max_id_dict is None:
266 267 268 269
        self.last_max_id_dict = PersistentMapping()
      self.last_max_id_dict.update(new_id_dict)

  security.declareProtected(Permissions.ModifyPortalContent,
270 271 272 273 274 275 276 277 278 279
      'rebuildSqlTable')
  def rebuildSqlTable(self):
    """
      After a mysql crash, it could be needed to restore values stored in
      zodb into mysql

      TODO : take into account the case where the value is stored every X
             generation 
    """
    portal = self.getPortalObject()
280 281
    portal.IdTool_zDropTable()
    portal.IdTool_zCreateEmptyTable()
282
    self._updateSqlTable()