Commit 434eb5de authored by Aurel's avatar Aurel

modify catalog to use archive if present when

catalogging/uncatalogging object
modify hot reindex for use whith archiving system


git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@16128 20353a03-c40f-0410-a6d1-a30d3c3de9de
parent 80fc64b9
......@@ -29,12 +29,13 @@ from SQLCatalog import Catalog, CatalogError
from AccessControl import ClassSecurityInfo, getSecurityManager
from AccessControl.DTML import RestrictedDTML
from Products.CMFCore.utils import getToolByName
from Products.ERP5Type.Cache import clearCache
import string, os, sys, types
import time
import urllib
from ZODB.POSException import ConflictError
from zLOG import LOG
from zLOG import LOG, ERROR, INFO
_marker = object()
......@@ -183,6 +184,7 @@ class ZCatalog(Folder, Persistent, Implicit):
destination_sql_catalog_id = None
hot_reindexing_state = None
default_sql_catalog_id = None
archive = None
manage_catalogAddRowForm = DTMLFile('dtml/catalogAddRowForm', globals())
manage_catalogFilter = DTMLFile( 'dtml/catalogFilter', globals() )
......@@ -250,13 +252,14 @@ class ZCatalog(Folder, Persistent, Implicit):
return HOT_REINDEXING_FINISHED_STATE
return value
def setHotReindexingState(self, state='', source_sql_catalog_id=None, destination_sql_catalog_id=None):
def setHotReindexingState(self, state='', source_sql_catalog_id=None, destination_sql_catalog_id=None, archive=None):
"""
Set the state of hot reindexing.
Do not use setProperty because the state should not modified from the ZMI directly.
It must be maintained very carefully.
"""
#LOG("setHotReindexingState call", 300, state)
if source_sql_catalog_id is None:
source_sql_catalog_id = self.default_sql_catalog_id
......@@ -264,11 +267,13 @@ class ZCatalog(Folder, Persistent, Implicit):
self.hot_reindexing_state = None
self.source_sql_catalog_id = None
self.destination_sql_catalog_id = None
self.archive = None
elif state == HOT_REINDEXING_RECORDING_STATE or \
state == HOT_REINDEXING_DOUBLE_INDEXING_STATE:
self.hot_reindexing_state = state
self.source_sql_catalog_id = source_sql_catalog_id
self.destination_sql_catalog_id = destination_sql_catalog_id
self.archive = archive
else:
raise CatalogError, 'unknown hot reindexing state %s' % state
......@@ -283,6 +288,7 @@ class ZCatalog(Folder, Persistent, Implicit):
skin_selection_dict=skin_selection_dict,
sql_connection_id_dict=sql_connection_id_dict)
self.setHotReindexingState(state=HOT_REINDEXING_FINISHED_STATE)
clearCache(cache_factory_list=('erp5_content_short',))
def cancelHotReindexing(self):
"""
......@@ -381,17 +387,19 @@ class ZCatalog(Folder, Persistent, Implicit):
LOG('exchangeDatabases skin_selection_dict:',0,skin_selection_dict)
if skin_selection_dict is not None:
LOG('exchangeDatabases skin_selection_dict:',0,'we will do manage_skinLayers')
#LOG('exchangeDatabases skin_selection_dict:',0,'we will do manage_skinLayers')
for skin_name, selection in self.portal_skins.getSkinPaths():
if skin_name in skin_selection_dict:
new_selection = tuple(skin_selection_dict[skin_name])
self.portal_skins.manage_skinLayers(skinpath = new_selection, skinname = skin_name, add_skin = 1)
LOG('exchangeDatabases sql_connection_id_dict :',0,sql_connection_id_dict)
if sql_connection_id_dict is not None:
self.changeSQLConnectionIds(self.portal_skins, sql_connection_id_dict)
def manage_hotReindexAll(self, source_sql_catalog_id,
destination_sql_catalog_id,
archive=None,
source_sql_connection_id_list=None,
destination_sql_connection_id_list=None,
skin_name_list=None,
......@@ -428,7 +436,7 @@ class ZCatalog(Folder, Persistent, Implicit):
"""
# Hot reindexing can only be runing once at a time on a system.
if self.hot_reindexing_state is not None:
raise CatalogError, 'hot reindexing process is already running'
raise CatalogError, 'hot reindexing process is already running %s -%s' %(self, self.hot_reindexing_state)
if source_sql_catalog_id == destination_sql_catalog_id:
raise CatalogError, 'Hot reindexing cannot be done with the same '\
......@@ -477,8 +485,8 @@ class ZCatalog(Folder, Persistent, Implicit):
portal = self.getPortalObject()
for id in portal.objectIds():
getUid = getattr(portal[id], 'getUid', None)
LOG('before getUid portal[id], id',0,id)
if getUid is not None:
if getUid is not None and id != "portal_uidhandler":
# XXX check adviced by yo, getUid is different for this tool
getUid() # Trigger the uid generation if none is set.
# Mark the hot reindex as begun. Each object indexed in the still-current
......@@ -486,7 +494,8 @@ class ZCatalog(Folder, Persistent, Implicit):
LOG('hotReindexObjectList', 0, 'Starting recording')
self.setHotReindexingState(HOT_REINDEXING_RECORDING_STATE,
source_sql_catalog_id=source_sql_catalog_id,
destination_sql_catalog_id=destination_sql_catalog_id)
destination_sql_catalog_id=destination_sql_catalog_id,
archive=archive)
# Clear the future catalog and start reindexing the site in it.
final_activity_tag = 'hot_reindex_last_ERP5Site_reindexAll_tag'
self.ERP5Site_reindexAll(sql_catalog_id=destination_sql_catalog_id,
......@@ -508,7 +517,8 @@ class ZCatalog(Folder, Persistent, Implicit):
# Once there is nothing to replay, databases are sync'ed, so the new
# catalog can become current.
self.activate(passive_commit=1,
after_method_id=('playBackRecordedObjectList'),
after_method_id=('playBackRecordedObjectList', 'runInventoryMethod'),
after_tag='runInventoryMethod',
priority=5).finishHotReindexing(
source_sql_catalog_id=source_sql_catalog_id,
destination_sql_catalog_id=destination_sql_catalog_id,
......@@ -676,17 +686,44 @@ class ZCatalog(Folder, Persistent, Implicit):
""" wrapper around catalog """
self.catalogObjectList([obj], sql_catalog_id=sql_catalog_id)
def catalogObjectList(self, object_list, sql_catalog_id=None, **kw):
def catalogObjectList(self, object_list, sql_catalog_id=None, disable_archive=0, **kw):
"""Catalog a list of objects.
"""
catalog = self.getSQLCatalog(sql_catalog_id)
default_catalog = self.getSQLCatalog()
hot_reindexing = (self.hot_reindexing_state is not None) and \
(catalog is not None) and \
(self.source_sql_catalog_id == catalog.id)
archiving = self.archive is not None
wrapped_object_list = []
failed_object_list = []
url_list = []
if getattr(self, "portal_archives", None) is not None:
archive_list = self.portal_archives.getArchiveList()
else:
archive_list = []
catalog_dict = {}
# Create archive obj list if necessary
if archiving:
archive_obj_list = [self.archive,]
for archive_path in archive_list:
try:
archive = self.unrestrictedTraverse(archive_path)
except KeyError:
continue
if archive.getCatalogId() == self.destination_sql_catalog_id:
archive_obj_list.append(archive)
else:
archive_obj_list = []
for archive_path in archive_list:
try:
archive = self.unrestrictedTraverse(archive_path)
except KeyError:
continue
archive_obj_list.append(archive)
# Construct list of object to catalog
for obj in object_list:
if hot_reindexing:
try:
......@@ -699,24 +736,73 @@ class ZCatalog(Folder, Persistent, Implicit):
url = '/'.join(url())
url_list.append(url)
goto_current_catalog = 0
if disable_archive == 0 and (archiving or (len(archive_obj_list) > 0 and \
(sql_catalog_id == default_catalog.id or \
sql_catalog_id is None))):
# check in which archive object must go
catalog_id = None
for archive in archive_obj_list:
if archive.test(obj) is True:
goto_current_catalog = 0
catalog_id = archive.getCatalogId()
priority = archive.getPriority()
if catalog_dict.has_key(catalog_id):
catalog_dict[catalog_id]['obj'].append(obj)
else:
catalog_dict[catalog_id] = {'priority' : priority, 'obj' : [obj,]}
if catalog_id is None: # or sql_catalog_id is None:
# at least put object in current catalog if no archive match
goto_current_catalog = 1
else:
goto_current_catalog = 1
if goto_current_catalog == 1:
try:
obj = self.wrapObject(obj, sql_catalog_id=sql_catalog_id)
# wrap object only when sure it will be reindex now
# thus security uid is also reindex
wrap_obj = self.wrapObject(obj, sql_catalog_id=sql_catalog_id)
except ConflictError:
raise
except:
LOG('WARNING ZSQLCatalog', 0, 'wrapObject failed on the object %r' % (obj,), error=sys.exc_info())
failed_object_list.append(obj)
wrapped_object_list.append(wrap_obj)
# run activity or execute for each archive depending on priority
if len(catalog_dict):
for catalog_id in catalog_dict.keys():
d = catalog_dict[catalog_id]
if hot_reindexing and self.hot_reindexing_state == HOT_REINDEXING_DOUBLE_INDEXING_STATE and \
self.destination_sql_catalog_id == catalog_id:
destination_catalog = self.getSQLCatalog(self.destination_sql_catalog_id)
# wrap all objects
wrapped_object_list_2 = []
for obj in d['obj']:
try:
wrap_obj = self.wrapObject(obj, sql_catalog_id=catalog_id)
except ConflictError:
raise
except:
LOG('WARNING ZSQLCatalog', 0, 'wrapObject failed on the object %r' % (obj,), error=sys.exc_info())
failed_object_list.append(obj)
wrapped_object_list_2.append(wrap_obj)
# reindex objects in destination catalog
destination_catalog.catalogObjectList(wrapped_object_list_2, **kw)
else:
wrapped_object_list.append(obj)
for obj in d['obj']:
obj.reindexObject(sql_catalog_id=catalog_id, activate_kw = \
{'priority': d['priority']}, disable_archive=1, **kw)
if catalog is not None:
catalog.catalogObjectList(wrapped_object_list,**kw)
if len(wrapped_object_list):
catalog.catalogObjectList(wrapped_object_list, **kw)
if hot_reindexing:
destination_catalog = self.getSQLCatalog(self.destination_sql_catalog_id)
if destination_catalog.id != catalog.id:
if self.hot_reindexing_state == HOT_REINDEXING_RECORDING_STATE:
destination_catalog.recordObjectList(url_list, 1)
else:
if len(wrapped_object_list):
destination_catalog.catalogObjectList(wrapped_object_list,**kw)
object_list[:] = failed_object_list[:]
......@@ -724,12 +810,26 @@ class ZCatalog(Folder, Persistent, Implicit):
def uncatalog_object(self, uid=None,path=None, sql_catalog_id=None):
""" wrapper around catalog """
catalog = self.getSQLCatalog(sql_catalog_id)
if uid is None:
raise TypeError, "sorry uncatalog_object supports only uid"
if catalog is not None:
catalog.uncatalogObject(uid=uid,path=path)
if getattr(self, "portal_archives", None) is not None:
archive_list = self.portal_archives.getArchiveList()
else:
archive_list = []
catalog_id = None
if len(archive_list) and sql_catalog_id is None:
for archive_path in archive_list:
try:
archive = self.unrestrictedTraverse(archive_path)
except KeyError:
continue
catalog_id = archive.getCatalogId()
archive.activate(priority=archive.getPriority()).uncatalogObject(uid=uid,path=path,
sql_catalog_id=catalog_id)
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None and catalog_id is None:
catalog.uncatalogObject(uid=uid,path=path)
if self.hot_reindexing_state is not None and self.source_sql_catalog_id == catalog.id:
destination_catalog = self.getSQLCatalog(self.destination_sql_catalog_id)
if destination_catalog.id != catalog.id:
......@@ -738,6 +838,7 @@ class ZCatalog(Folder, Persistent, Implicit):
else:
destination_catalog.uncatalogObject(uid=uid)
def beforeUncatalogObject(self, uid=None,path=None, sql_catalog_id=None):
""" wrapper around catalog """
catalog = self.getSQLCatalog(sql_catalog_id)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment