From 434eb5deeb2f31d3a99e434013e8ff0635b41ab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Calonne?= <aurel@nexedi.com> Date: Thu, 6 Sep 2007 15:19:17 +0000 Subject: [PATCH] modify catalog to use archive if present when catalogging/uncatalogging object modify hot reindex for use whith archiving system git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@16128 20353a03-c40f-0410-a6d1-a30d3c3de9de --- product/ZSQLCatalog/ZSQLCatalog.py | 171 +++++++++++++++++++++++------ 1 file changed, 136 insertions(+), 35 deletions(-) diff --git a/product/ZSQLCatalog/ZSQLCatalog.py b/product/ZSQLCatalog/ZSQLCatalog.py index 03693e6564..1d5d40054e 100644 --- a/product/ZSQLCatalog/ZSQLCatalog.py +++ b/product/ZSQLCatalog/ZSQLCatalog.py @@ -29,12 +29,13 @@ from SQLCatalog import Catalog, CatalogError from AccessControl import ClassSecurityInfo, getSecurityManager from AccessControl.DTML import RestrictedDTML from Products.CMFCore.utils import getToolByName +from Products.ERP5Type.Cache import clearCache import string, os, sys, types import time import urllib from ZODB.POSException import ConflictError -from zLOG import LOG +from zLOG import LOG, ERROR, INFO _marker = object() @@ -183,7 +184,8 @@ class ZCatalog(Folder, Persistent, Implicit): destination_sql_catalog_id = None hot_reindexing_state = None default_sql_catalog_id = None - + archive = None + manage_catalogAddRowForm = DTMLFile('dtml/catalogAddRowForm', globals()) manage_catalogFilter = DTMLFile( 'dtml/catalogFilter', globals() ) manage_catalogView = DTMLFile('dtml/catalogView',globals()) @@ -250,13 +252,14 @@ class ZCatalog(Folder, Persistent, Implicit): return HOT_REINDEXING_FINISHED_STATE return value - def setHotReindexingState(self, state='', source_sql_catalog_id=None, destination_sql_catalog_id=None): + def setHotReindexingState(self, state='', source_sql_catalog_id=None, destination_sql_catalog_id=None, archive=None): """ Set the state of hot reindexing. Do not use setProperty because the state should not modified from the ZMI directly. It must be maintained very carefully. """ + #LOG("setHotReindexingState call", 300, state) if source_sql_catalog_id is None: source_sql_catalog_id = self.default_sql_catalog_id @@ -264,11 +267,13 @@ class ZCatalog(Folder, Persistent, Implicit): self.hot_reindexing_state = None self.source_sql_catalog_id = None self.destination_sql_catalog_id = None + self.archive = None elif state == HOT_REINDEXING_RECORDING_STATE or \ state == HOT_REINDEXING_DOUBLE_INDEXING_STATE: self.hot_reindexing_state = state self.source_sql_catalog_id = source_sql_catalog_id self.destination_sql_catalog_id = destination_sql_catalog_id + self.archive = archive else: raise CatalogError, 'unknown hot reindexing state %s' % state @@ -283,6 +288,7 @@ class ZCatalog(Folder, Persistent, Implicit): skin_selection_dict=skin_selection_dict, sql_connection_id_dict=sql_connection_id_dict) self.setHotReindexingState(state=HOT_REINDEXING_FINISHED_STATE) + clearCache(cache_factory_list=('erp5_content_short',)) def cancelHotReindexing(self): """ @@ -381,23 +387,25 @@ class ZCatalog(Folder, Persistent, Implicit): LOG('exchangeDatabases skin_selection_dict:',0,skin_selection_dict) if skin_selection_dict is not None: - LOG('exchangeDatabases skin_selection_dict:',0,'we will do manage_skinLayers') + #LOG('exchangeDatabases skin_selection_dict:',0,'we will do manage_skinLayers') for skin_name, selection in self.portal_skins.getSkinPaths(): if skin_name in skin_selection_dict: new_selection = tuple(skin_selection_dict[skin_name]) self.portal_skins.manage_skinLayers(skinpath = new_selection, skinname = skin_name, add_skin = 1) + LOG('exchangeDatabases sql_connection_id_dict :',0,sql_connection_id_dict) if sql_connection_id_dict is not None: self.changeSQLConnectionIds(self.portal_skins, sql_connection_id_dict) def manage_hotReindexAll(self, source_sql_catalog_id, - destination_sql_catalog_id, - source_sql_connection_id_list=None, - destination_sql_connection_id_list=None, - skin_name_list=None, - skin_selection_list=None, - update_destination_sql_catalog=None, - REQUEST=None, RESPONSE=None): + destination_sql_catalog_id, + archive=None, + source_sql_connection_id_list=None, + destination_sql_connection_id_list=None, + skin_name_list=None, + skin_selection_list=None, + update_destination_sql_catalog=None, + REQUEST=None, RESPONSE=None): """ Starts a hot reindexing. @@ -428,7 +436,7 @@ class ZCatalog(Folder, Persistent, Implicit): """ # Hot reindexing can only be runing once at a time on a system. if self.hot_reindexing_state is not None: - raise CatalogError, 'hot reindexing process is already running' + raise CatalogError, 'hot reindexing process is already running %s -%s' %(self, self.hot_reindexing_state) if source_sql_catalog_id == destination_sql_catalog_id: raise CatalogError, 'Hot reindexing cannot be done with the same '\ @@ -477,8 +485,8 @@ class ZCatalog(Folder, Persistent, Implicit): portal = self.getPortalObject() for id in portal.objectIds(): getUid = getattr(portal[id], 'getUid', None) - LOG('before getUid portal[id], id',0,id) - if getUid is not None: + if getUid is not None and id != "portal_uidhandler": + # XXX check adviced by yo, getUid is different for this tool getUid() # Trigger the uid generation if none is set. # Mark the hot reindex as begun. Each object indexed in the still-current @@ -486,10 +494,11 @@ class ZCatalog(Folder, Persistent, Implicit): LOG('hotReindexObjectList', 0, 'Starting recording') self.setHotReindexingState(HOT_REINDEXING_RECORDING_STATE, source_sql_catalog_id=source_sql_catalog_id, - destination_sql_catalog_id=destination_sql_catalog_id) + destination_sql_catalog_id=destination_sql_catalog_id, + archive=archive) # Clear the future catalog and start reindexing the site in it. final_activity_tag = 'hot_reindex_last_ERP5Site_reindexAll_tag' - self.ERP5Site_reindexAll(sql_catalog_id=destination_sql_catalog_id, + self.ERP5Site_reindexAll(sql_catalog_id=destination_sql_catalog_id, final_activity_tag=final_activity_tag, clear_catalog=1, passive_commit=1) @@ -508,7 +517,8 @@ class ZCatalog(Folder, Persistent, Implicit): # Once there is nothing to replay, databases are sync'ed, so the new # catalog can become current. self.activate(passive_commit=1, - after_method_id=('playBackRecordedObjectList'), + after_method_id=('playBackRecordedObjectList', 'runInventoryMethod'), + after_tag='runInventoryMethod', priority=5).finishHotReindexing( source_sql_catalog_id=source_sql_catalog_id, destination_sql_catalog_id=destination_sql_catalog_id, @@ -676,17 +686,44 @@ class ZCatalog(Folder, Persistent, Implicit): """ wrapper around catalog """ self.catalogObjectList([obj], sql_catalog_id=sql_catalog_id) - def catalogObjectList(self, object_list, sql_catalog_id=None, **kw): + def catalogObjectList(self, object_list, sql_catalog_id=None, disable_archive=0, **kw): """Catalog a list of objects. """ catalog = self.getSQLCatalog(sql_catalog_id) + default_catalog = self.getSQLCatalog() hot_reindexing = (self.hot_reindexing_state is not None) and \ (catalog is not None) and \ (self.source_sql_catalog_id == catalog.id) - + archiving = self.archive is not None wrapped_object_list = [] failed_object_list = [] url_list = [] + if getattr(self, "portal_archives", None) is not None: + archive_list = self.portal_archives.getArchiveList() + else: + archive_list = [] + + catalog_dict = {} + + # Create archive obj list if necessary + if archiving: + archive_obj_list = [self.archive,] + for archive_path in archive_list: + try: + archive = self.unrestrictedTraverse(archive_path) + except KeyError: + continue + if archive.getCatalogId() == self.destination_sql_catalog_id: + archive_obj_list.append(archive) + else: + archive_obj_list = [] + for archive_path in archive_list: + try: + archive = self.unrestrictedTraverse(archive_path) + except KeyError: + continue + archive_obj_list.append(archive) + # Construct list of object to catalog for obj in object_list: if hot_reindexing: try: @@ -698,38 +735,101 @@ class ZCatalog(Folder, Persistent, Implicit): ) url = '/'.join(url()) url_list.append(url) - - try: - obj = self.wrapObject(obj, sql_catalog_id=sql_catalog_id) - except ConflictError: - raise - except: - LOG('WARNING ZSQLCatalog', 0, 'wrapObject failed on the object %r' % (obj,), error=sys.exc_info()) - failed_object_list.append(obj) + + goto_current_catalog = 0 + if disable_archive == 0 and (archiving or (len(archive_obj_list) > 0 and \ + (sql_catalog_id == default_catalog.id or \ + sql_catalog_id is None))): + # check in which archive object must go + catalog_id = None + for archive in archive_obj_list: + if archive.test(obj) is True: + goto_current_catalog = 0 + catalog_id = archive.getCatalogId() + priority = archive.getPriority() + if catalog_dict.has_key(catalog_id): + catalog_dict[catalog_id]['obj'].append(obj) + else: + catalog_dict[catalog_id] = {'priority' : priority, 'obj' : [obj,]} + if catalog_id is None: # or sql_catalog_id is None: + # at least put object in current catalog if no archive match + goto_current_catalog = 1 else: - wrapped_object_list.append(obj) - + goto_current_catalog = 1 + if goto_current_catalog == 1: + try: + # wrap object only when sure it will be reindex now + # thus security uid is also reindex + wrap_obj = self.wrapObject(obj, sql_catalog_id=sql_catalog_id) + except ConflictError: + raise + except: + LOG('WARNING ZSQLCatalog', 0, 'wrapObject failed on the object %r' % (obj,), error=sys.exc_info()) + failed_object_list.append(obj) + wrapped_object_list.append(wrap_obj) + + # run activity or execute for each archive depending on priority + if len(catalog_dict): + for catalog_id in catalog_dict.keys(): + d = catalog_dict[catalog_id] + if hot_reindexing and self.hot_reindexing_state == HOT_REINDEXING_DOUBLE_INDEXING_STATE and \ + self.destination_sql_catalog_id == catalog_id: + destination_catalog = self.getSQLCatalog(self.destination_sql_catalog_id) + # wrap all objects + wrapped_object_list_2 = [] + for obj in d['obj']: + try: + wrap_obj = self.wrapObject(obj, sql_catalog_id=catalog_id) + except ConflictError: + raise + except: + LOG('WARNING ZSQLCatalog', 0, 'wrapObject failed on the object %r' % (obj,), error=sys.exc_info()) + failed_object_list.append(obj) + wrapped_object_list_2.append(wrap_obj) + # reindex objects in destination catalog + destination_catalog.catalogObjectList(wrapped_object_list_2, **kw) + else: + for obj in d['obj']: + obj.reindexObject(sql_catalog_id=catalog_id, activate_kw = \ + {'priority': d['priority']}, disable_archive=1, **kw) + if catalog is not None: - catalog.catalogObjectList(wrapped_object_list,**kw) + if len(wrapped_object_list): + catalog.catalogObjectList(wrapped_object_list, **kw) if hot_reindexing: destination_catalog = self.getSQLCatalog(self.destination_sql_catalog_id) if destination_catalog.id != catalog.id: if self.hot_reindexing_state == HOT_REINDEXING_RECORDING_STATE: destination_catalog.recordObjectList(url_list, 1) else: - destination_catalog.catalogObjectList(wrapped_object_list,**kw) - + if len(wrapped_object_list): + destination_catalog.catalogObjectList(wrapped_object_list,**kw) + object_list[:] = failed_object_list[:] def uncatalog_object(self, uid=None,path=None, sql_catalog_id=None): """ wrapper around catalog """ - catalog = self.getSQLCatalog(sql_catalog_id) if uid is None: raise TypeError, "sorry uncatalog_object supports only uid" - if catalog is not None: - catalog.uncatalogObject(uid=uid,path=path) + if getattr(self, "portal_archives", None) is not None: + archive_list = self.portal_archives.getArchiveList() + else: + archive_list = [] + catalog_id = None + if len(archive_list) and sql_catalog_id is None: + for archive_path in archive_list: + try: + archive = self.unrestrictedTraverse(archive_path) + except KeyError: + continue + catalog_id = archive.getCatalogId() + archive.activate(priority=archive.getPriority()).uncatalogObject(uid=uid,path=path, + sql_catalog_id=catalog_id) + catalog = self.getSQLCatalog(sql_catalog_id) + if catalog is not None and catalog_id is None: + catalog.uncatalogObject(uid=uid,path=path) if self.hot_reindexing_state is not None and self.source_sql_catalog_id == catalog.id: destination_catalog = self.getSQLCatalog(self.destination_sql_catalog_id) if destination_catalog.id != catalog.id: @@ -738,6 +838,7 @@ class ZCatalog(Folder, Persistent, Implicit): else: destination_catalog.uncatalogObject(uid=uid) + def beforeUncatalogObject(self, uid=None,path=None, sql_catalog_id=None): """ wrapper around catalog """ catalog = self.getSQLCatalog(sql_catalog_id) -- 2.30.9