Skip to content

erp5
erp5
Commit cacdc675 authored by Yoshinori Okuji's avatar Yoshinori Okuji
Browse files
Options

Big changes for supporting hot reindexing. 

Most features in ZSQLCatalog are moved to SQLCatalog.
ZSQLCatalog may contain multiple SQL Catalogs.
ZSQLCatalog supports filtering without ERP5Catalog.
A workaround for non-CMF sites is added.
Exporting/Importing properties is supported in the form of XML.
Queued indexing is supported.


git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@2002 20353a03-c40f-0410-a6d1-a30d3c3de9de
parent 55db6c81
Hide whitespace changes
Inline Side-by-side
Showing with 1403 additions and 423 deletions
product/ZSQLCatalog/SQLCatalog.py
View file @ cacdc675
......@@ -15,23 +15,62 @@
from Persistence import Persistent
import Acquisition
import ExtensionClass
import Globals
from Globals import DTMLFile, PersistentMapping
from string import lower, split, join
from thread import get_ident
from OFS.Folder import Folder
from AccessControl import ClassSecurityInfo, getSecurityManager
from BTrees.OIBTree import OIBTree
from DateTime import DateTime
from Products.PluginIndexes.common.randid import randid
from Products.CMFCore.Expression import Expression
from Products.CMFCore.utils import getToolByName
from Acquisition import aq_parent, aq_inner, aq_base, aq_self
from zLOG import LOG
import time
import sys
import urllib
import string
from cStringIO import StringIO
from xml.dom.minidom import parse, parseString, getDOMImplementation
from xml.sax.saxutils import escape, quoteattr
try:
from Products.PageTemplates.Expressions import SecureModuleImporter
from Products.CMFCore.Expression import Expression
from Products.PageTemplates.Expressions import getEngine
from Products.CMFCore.utils import getToolByName
withCMF = 1
except ImportError:
withCMF = 0
UID_BUFFER_SIZE = 900
MAX_UID_BUFFER_SIZE = 20000
class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
valid_method_meta_type_list = ('Z SQL Method', 'Script (Python)')
manage_addSQLCatalogForm=DTMLFile('dtml/addSQLCatalog',globals())
def manage_addSQLCatalog(self, id, title,
vocab_id='create_default_catalog_',
REQUEST=None):
"""Add a Catalog object
"""
id=str(id)
title=str(title)
vocab_id=str(vocab_id)
if vocab_id == 'create_default_catalog_':
vocab_id = None
c=Catalog(id, title, self)
self._setObject(id, c)
if REQUEST is not None:
return self.manage_main(self, REQUEST,update_menu=1)
class Catalog(Folder, Persistent, Acquisition.Implicit, ExtensionClass.Base):
""" An Object Catalog
An Object Catalog maintains a table of object metadata, and a
......@@ -53,13 +92,365 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
- optmization: indexing objects should be deferred
until timeout value or end of transaction
"""
meta_type = "SQLCatalog"
icon='misc_/ZCatalog/ZCatalog.gif' # FIXME: use a different icon
security = ClassSecurityInfo()
manage_options = (
{'label': 'Contents', # TAB: Contents
'action': 'manage_main',
'help': ('OFSP','ObjectManager_Contents.stx')},
{'label': 'Catalog', # TAB: Catalogged Objects
'action': 'manage_catalogView',
'target': 'manage_main',
'help':('ZCatalog','ZCatalog_Cataloged-Objects.stx')},
{'label': 'Properties', # TAB: Properties
'action': 'manage_propertiesForm',
'help': ('OFSP','Properties.stx')},
{'label': 'Filter', # TAB: Filter
'action': 'manage_catalogFilter',},
{'label': 'Find Objects', # TAB: Find Objects
'action': 'manage_catalogFind',
'target':'manage_main',
'help':('ZCatalog','ZCatalog_Find-Items-to-ZCatalog.stx')},
{'label': 'Advanced', # TAB: Advanced
'action': 'manage_catalogAdvanced',
'target':'manage_main',
'help':('ZCatalog','ZCatalog_Advanced.stx')},
{'label': 'Undo', # TAB: Undo
'action': 'manage_UndoForm',
'help': ('OFSP','Undo.stx')},
{'label': 'Security', # TAB: Security
'action': 'manage_access',
'help': ('OFSP','Security.stx')},
{'label': 'Ownership', # TAB: Ownership
'action': 'manage_owner',
'help': ('OFSP','Ownership.stx'),}
)
__ac_permissions__=(
('Manage ZCatalog Entries',
['manage_catalogObject', 'manage_uncatalogObject',
'manage_catalogView', 'manage_catalogFind',
'manage_catalogSchema', 'manage_catalogFilter',
'manage_catalogAdvanced', 'manage_objectInformation',
'manage_catalogReindex', 'manage_catalogFoundItems',
'manage_catalogClear', 'manage_editSchema',
'manage_reindexIndex', 'manage_main',
'manage_editFilter',
'manage_exportProperties', 'manage_importProperties',
],
['Manager']),
('Search ZCatalog',
['searchResults', '__call__', 'uniqueValuesFor',
'getpath', 'schema', 'names', 'columns', 'indexes', 'index_objects',
'all_meta_types', 'valid_roles', 'resolve_url',
'getobject', 'getObject', 'getObjectList', 'getCatalogSearchTableIds',
'getCatalogSearchResultKeys', 'getFilterableMethodList', ],
['Anonymous', 'Manager']),
('Import/Export objects',
['manage_exportProperties', 'manage_importProperties', ],
['Manager']),
)
_properties = (
{ 'id' : 'title',
'description' : 'The title of this catalog',
'type' : 'string',
'mode' : 'w' },
# Z SQL Methods
{ 'id' : 'sql_catalog_produce_reserved',
'description' : 'A method to produce new uid values in advance',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_clear_reserved',
'description' : 'A method to clear reserved uid values',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_object',
'description' : 'Methods to be called to catalog an object',
'type' : 'multiple selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_uncatalog_object',
'description' : 'Methods to be called to uncatalog an object',
'type' : 'multiple selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_update_object',
'description' : 'Methods will be called to updat a catalogued object',
'type' : 'multiple selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_clear_catalog',
'description' : 'The methods which should be called to clear the catalog',
'type' : 'multiple selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_object_list',
'description' : 'Methods to be called to catalog the list of objects',
'type' : 'multiple selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_record_catalog_object',
'description' : 'Method to record catalog information',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_record_uncatalog_object',
'description' : 'Method to record uncatalog information',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_read_recorded_object_list',
'description' : 'Method to get recorded information',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_delete_recorded_object_list',
'description' : 'Method to delete recorded information',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_search_results',
'description' : 'Main method to search the catalog',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_search_tables',
'description' : 'Tables to join in the result',
'type' : 'multiple selection',
'select_variable' : 'getTableIds',
'mode' : 'w' },
{ 'id' : 'sql_search_result_keys',
'description' : 'Keys to display in the result',
'type' : 'multiple selection',
'select_variable' : 'getResultColumnIds',
'mode' : 'w' },
{ 'id' : 'sql_count_results',
'description' : 'Main method to search the catalog',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_getitem_by_path',
'description' : 'Get a catalog brain by path',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_getitem_by_uid',
'description' : 'Get a catalog brain by uid',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_tables',
'description' : 'Method to get the main catalog tables',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_schema',
'description' : 'Method to get the main catalog schema',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_unique_values',
'description' : 'Find unique disctinct values in a column',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_paths',
'description' : 'List all object paths in catalog',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_keyword_search_keys',
'description' : 'Columns which should be considered as full text search',
'type' : 'multiple selection',
'select_variable' : 'getColumnIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_full_text_search_keys',
'description' : 'Columns which should be considered as full text search',
'type' : 'multiple selection',
'select_variable' : 'getColumnIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_request_keys',
'description' : 'Columns which should be ignore in the REQUEST in order to accelerate caching',
'type' : 'multiple selection',
'select_variable' : 'getColumnIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_multivalue_keys',
'description' : 'Keys which hold multiple values',
'type' : 'multiple selection',
'select_variable' : 'getColumnIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_topic_search_keys',
'description' : 'Columns which should be considered as topic index',
'type' : 'lines',
'mode' : 'w' },
{ 'id' : 'sql_catalog_related_keys',
'title' : 'Related keys',
'description' : 'Additional columns obtained through joins',
'type' : 'lines',
'mode' : 'w' },
)
sql_catalog_produce_reserved = 'z_produce_reserved_uid_list'
sql_catalog_clear_reserved = 'z_clear_reserved'
sql_catalog_object = ('z_catalog_object',)
sql_uncatalog_object = ('z_uncatalog_object',)
sql_update_object = ('z_update_object',)
sql_clear_catalog = ('z_drop_catalog', 'z_create_catalog')
sql_catalog_object_list = ('z_catalog_object_list',)
sql_record_catalog_object = 'z_record_catalog_object'
sql_record_uncatalog_object = 'z_record_uncatalog_object'
sql_read_recorded_object_list = 'z_read_recorded_object_list'
sql_delete_recorded_object_list = 'z_delete_recorded_object_list'
sql_search_results = 'z_search_results'
sql_count_results = 'z_count_results'
sql_getitem_by_path = 'z_getitem_by_path'
sql_getitem_by_uid = 'z_getitem_by_uid'
sql_catalog_tables = 'z_catalog_tables'
sql_search_tables = ('catalog',)
sql_catalog_schema = 'z_catalog_schema'
sql_unique_values = 'z_unique_values'
sql_catalog_paths = 'z_catalog_paths'
sql_catalog_keyword_search_keys = ('Description', 'Title', 'SearchableText')
sql_catalog_full_text_search_keys = ()
sql_catalog_request_keys = ()
sql_search_result_keys = ()
sql_catalog_topic_search_keys = ()
sql_catalog_multivalue_keys = ()
sql_catalog_related_keys = ()
_after_clear_reserved = 0
def __init__(self):
manage_catalogView = DTMLFile('dtml/catalogView',globals())
manage_catalogFilter = DTMLFile('dtml/catalogFilter',globals())
manage_catalogFind = DTMLFile('dtml/catalogFind',globals())
manage_catalogAdvanced = DTMLFile('dtml/catalogAdvanced', globals())
def __init__(self, id, title='', container=None):
if container is not None:
self=self.__of__(container)
self.id=id
self.title=title
self.schema = {} # mapping from attribute name to column
self.names = {} # mapping from column to attribute name
self.indexes = {} # empty mapping
def manage_exportProperties(self, REQUEST=None, RESPONSE=None):
"""
Export properties to an XML file.
"""
f = StringIO()
f.write('<?xml version="1.0"?>\n<SQLCatalogData>\n')
id_list = self.propertyIds()
for id in id_list:
value = self.getProperty(id)
if value is None:
# What is this? Not used?
continue
if type(value) == type(""):
f.write(' <property id=%s type="str">%s</property>\n' % (quoteattr(id), escape(value)))
elif type(value) == type(()):
f.write(' <property id=%s type="tuple">\n' % quoteattr(id))
for item in value:
if type(item) == type(""):
f.write(' <item type="str">%s</item>\n' % escape(item))
else:
# Ignore the other types at the moment.
pass
f.write(' </property>\n')
else:
# Ignore the other types at the moment.
pass
f.write('</SQLCatalogData>\n')
if RESPONSE is not None:
RESPONSE.setHeader('Content-type','application/data')
RESPONSE.setHeader('Content-Disposition',
'inline;filename=properties.xml')
return f.getvalue()
def manage_importProperties(self, file):
"""
Import properties from an XML file.
"""
f = open(file)
try:
doc = parse(f)
root = doc.documentElement
try:
for prop in root.getElementsByTagName("property"):
id = prop.getAttribute("id")
type = prop.getAttribute("type")
if not id or not hasattr(self, id):
raise CatalogError, 'unknown property id %r' (id,)
if type not in ('str', 'tuple'):
raise CatalogError, 'unknown property type %r' (type,)
if type == 'str':
value = ''
for text in prop.childNodes:
if text.nodeType == text.TEXT_NODE:
value = text.data
break
else:
value = []
for item in prop.getElementsByTagName("item"):
item_type = item.getAttribute("type")
if item_type != 'str':
raise CatalogError, 'unknown item type %r' (item_type,)
for text in item.childNodes:
if text.nodeType == text.TEXT_NODE:
value.append(text.data)
break
value = tuple(value)
setattr(self, id, value)
finally:
doc.unlink()
finally:
f.close()
def _clearSecurityCache(self):
self.security_uid_dict = OIBTree()
self.security_uid_index = 0
security.declarePrivate('getSecurityUid')
def getSecurityUid(self, object):
"""
Cache a uid for each security permission
We try to create a unique security (to reduce number of lines)
and to assign security only to root document
"""
# Get security information
allowed_roles_and_users = object.allowedRolesAndUsers()
# Sort it
allowed_roles_and_users = list(allowed_roles_and_users)
allowed_roles_and_users.sort()
allowed_roles_and_users = tuple(allowed_roles_and_users)
# Make sure no diplicates
if not hasattr(aq_base(self), 'security_uid_dict'):
self._clearSecurityCache()
if self.security_uid_dict.has_key(allowed_roles_and_users):
return (self.security_uid_dict[allowed_roles_and_users], None)
self.security_uid_index = self.security_uid_index + 1
self.security_uid_dict[allowed_roles_and_users] = self.security_uid_index
return (self.security_uid_index, allowed_roles_and_users)
def clear(self):
"""
Clears the catalog by calling a list of methods
......@@ -70,8 +461,10 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
try:
method()
except:
LOG('SQLCatalog Warning: could not clear catalog', 0, method_name, error=sys.exc_info())
pass
self._after_clear_reserved = 1
self._clearSecurityCache()
def clearReserved(self):
"""
......@@ -113,6 +506,23 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
self.schema = schema
self.names = names
def getCatalogSearchTableIds(self):
"""Return selected tables of catalog which are used in JOIN.
catalaog is always first
"""
search_tables = self.sql_search_tables
if len(search_tables) > 0:
if search_tables[0] != 'catalog':
result = ['catalog']
for t in search_tables:
if t != 'catalog':
result.append(t)
self.sql_search_tables = result
else:
self.sql_search_tables = ['catalog']
return self.sql_search_tables
def getColumnIds(self):
"""
Calls the show column method and returns dictionnary of
......@@ -239,6 +649,99 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
else:
raise CatalogError("Could not retrieve new uid")
def manage_catalogObject(self, REQUEST, RESPONSE, URL1, urls=None):
""" index Zope object(s) that 'urls' point to """
if urls:
if isinstance(urls, types.StringType):
urls=(urls,)
for url in urls:
obj = self.resolve_path(url)
if not obj:
obj = self.resolve_url(url, REQUEST)
if obj is not None:
self.aq_parent.catalog_object(obj, url, sql_catalog_id=self.id)
RESPONSE.redirect(URL1 + '/manage_catalogView?manage_tabs_message=Object%20Cataloged')
def manage_uncatalogObject(self, REQUEST, RESPONSE, URL1, urls=None):
""" removes Zope object(s) 'urls' from catalog """
if urls:
if isinstance(urls, types.StringType):
urls=(urls,)
for url in urls:
self.aq_parent.uncatalog_object(url, sql_catalog_id=self.id)
RESPONSE.redirect(URL1 + '/manage_catalogView?manage_tabs_message=Object%20Uncataloged')
def manage_catalogReindex(self, REQUEST, RESPONSE, URL1):
""" clear the catalog, then re-index everything """
elapse = time.time()
c_elapse = time.clock()
self.aq_parent.refreshCatalog(clear=1, sql_catalog_id=self.id)
elapse = time.time() - elapse
c_elapse = time.clock() - c_elapse
RESPONSE.redirect(URL1 +
'/manage_catalogAdvanced?manage_tabs_message=' +
urllib.quote('Catalog Updated<br>'
'Total time: %s<br>'
'Total CPU time: %s' % (`elapse`, `c_elapse`)))
def manage_catalogClear(self, REQUEST=None, RESPONSE=None, URL1=None, sql_catalog_id=None):
""" clears the whole enchilada """
self.clear()
if RESPONSE and URL1:
RESPONSE.redirect(URL1 + '/manage_catalogAdvanced?manage_tabs_message=Catalog%20Cleared')
def manage_catalogClearReserved(self, REQUEST=None, RESPONSE=None, URL1=None):
""" clears the whole enchilada """
self.clearReserved()
if RESPONSE and URL1:
RESPONSE.redirect(URL1 + '/manage_catalogAdvanced?manage_tabs_message=Catalog%20Cleared')
def manage_catalogFoundItems(self, REQUEST, RESPONSE, URL2, URL1,
obj_metatypes=None,
obj_ids=None, obj_searchterm=None,
obj_expr=None, obj_mtime=None,
obj_mspec=None, obj_roles=None,
obj_permission=None):
""" Find object according to search criteria and Catalog them
"""
elapse = time.time()
c_elapse = time.clock()
words = 0
obj = REQUEST.PARENTS[1]
path = string.join(obj.getPhysicalPath(), '/')
results = self.aq_parent.ZopeFindAndApply(obj,
obj_metatypes=obj_metatypes,
obj_ids=obj_ids,
obj_searchterm=obj_searchterm,
obj_expr=obj_expr,
obj_mtime=obj_mtime,
obj_mspec=obj_mspec,
obj_permission=obj_permission,
obj_roles=obj_roles,
search_sub=1,
REQUEST=REQUEST,
apply_func=self.aq_parent.catalog_object,
apply_path=path,
sql_catalog_id=self.id)
elapse = time.time() - elapse
c_elapse = time.clock() - c_elapse
RESPONSE.redirect(URL1 + '/manage_catalogView?manage_tabs_message=' +
urllib.quote('Catalog Updated<br>Total time: %s<br>Total CPU time: %s' % (`elapse`, `c_elapse`)))
def catalogObject(self, object, path, is_object_moved=0):
"""
Adds an object to the Catalog by calling
......@@ -249,7 +752,12 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
'uid' is the unique Catalog identifier for this object
"""
#LOG('Catalog object:',0,str(path))
if withCMF:
zope_root = getToolByName(self, 'portal_url').getPortalObject().aq_parent
else:
zope_root = self.getPhysicalRoot()
root_indexable = int(getattr(zope_root,'isIndexable',1))
# Prepare the dictionnary of values
kw = {}
......@@ -268,6 +776,12 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
index = self.getUidForPath(path)
if index:
if (uid != index):
# The new database must not change the uid.
if self.hot_reindexing_state is not None and self.destination_sql_catalog_id == self.id:
self.uncatalogObject(path)
self.catalog_object(object, path, is_object_moved=is_object_moved,
sql_catalog_id=self.source_sql_catalog_id)
return self.catalogObject(object, path, is_object_moved=is_object_moved)
# Update uid attribute of object
uid = int(index)
#LOG("Write Uid",0, "uid %s index %s" % (uid, index))
......@@ -275,7 +789,8 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
# We will check if there is an filter on this
# method, if so we may not call this zsqlMethod
# for this object
for method_name in self.sql_update_object:
methods = self.sql_update_object
for method_name in methods:
if self.isMethodFiltered(method_name):
if self.filter_dict.has_key(method_name):
portal_type = object.getPortalType()
......@@ -316,7 +831,13 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
# LOG("Call SQL Method %s with args:" % method_name,0, str(kw))
# Alter row
#LOG("Call SQL Method %s with args:" % method_name,0, str(kw))
method(**kw)
try:
if root_indexable:
#LOG("Call SQL Method %s with args:" % method_name,0, str(kw))
method(**kw)
except:
LOG("SQLCatalog Warning: could not catalog object with method %s" % method_name,100, str(path))
raise
else:
# Get the appropriate SQL Method
# Lookup by path is required because of OFS Semantics
......@@ -346,7 +867,8 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
#LOG("SQLCatalog Warning: insert_catalog_line, case5",0,insert_catalog_line)
else:
index = uid
for method_name in self.sql_catalog_object:
methods = self.sql_catalog_object
for method_name in methods:
# We will check if there is an filter on this
# method, if so we may not call this zsqlMethod
# for this object
......@@ -384,10 +906,8 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
kw['path'] = path
kw['uid'] = index
kw['insert_catalog_line'] = insert_catalog_line
# Alter/Create row
# Alter/Create row
try:
zope_root = getToolByName(self, 'portal_url').getPortalObject().aq_parent
root_indexable = int(getattr(zope_root,'isIndexable',1))
if root_indexable:
#LOG("Call SQL Method %s with args:" % method_name,0, str(kw))
method(**kw)
......@@ -400,6 +920,95 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
# LOG("SQLCatalog Warning: could not catalog object with method %s" % method_name,
# 100,str(path))
# XXX It is better to merge this with catalogObject. Too much code duplication.
def catalogObjectList(self, object_list):
"""
Add objects to the Catalog by calling
all SQL methods and providing needed arguments.
Each element of 'object_list' is an object to be cataloged
'uid' is the unique Catalog identifier for this object
WARNING: This method assumes that currently all objects are being reindexed from scratch.
XXX: For now newUid is used to allocated UIDs. Is this good? Is it better to INSERT then SELECT?
"""
if withCMF:
zope_root = getToolByName(self, 'portal_url').getPortalObject().aq_parent
else:
zope_root = self.getPhysicalRoot()
root_indexable = int(getattr(zope_root,'isIndexable',1))
if not root_indexable:
return
for object in object_list:
if not hasattr(aq_base(object), 'uid'):
# XXX should do something better
raise RuntimeError, '%r does not have uid' % (object,)
methods = self.sql_catalog_object_list
for method_name in methods:
kw = {}
#LOG('catalogObjectList', 0, 'method_name = %s, self.isMethodFiltered(method_name) = %r, self.filter_dict.has_key(method_name) = %r' % (method_name, self.isMethodFiltered(method_name), self.filter_dict.has_key(method_name)))
if self.isMethodFiltered(method_name) and self.filter_dict.has_key(method_name):
catalogged_object_list = []
type_list = self.filter_dict[method_name]['type']
expression = self.filter_dict[method_name]['expression_instance']
#LOG('catalogObjectList', 0, 'method_name = %s, type_list = %r, expression = %r' % (method_name, type_list, expression))
for object in object_list:
# We will check if there is an filter on this
# method, if so we may not call this zsqlMethod
# for this object
portal_type = object.getPortalType()
if portal_type not in type_list:
continue
elif expression is not None:
econtext = self.getExpressionContext(object)
result = expression(econtext)
if not result:
continue
catalogged_object_list.append(object)
else:
catalogged_object_list = object_list
if len(catalogged_object_list) == 0:
continue
#LOG('catalogObjectList', 0, 'method_name = %s' % (method_name,))
method = getattr(self, method_name)
if method.meta_type == "Z SQL Method":
# Build the dictionnary of values
arguments = method.arguments_src
for arg in split(arguments):
value_list = []
for object in catalogged_object_list:
try:
value = getattr(object, arg)
if callable(value):
value = value()
value_list.append(value)
except:
#LOG("SQLCatalog Warning: Callable value could not be called",0,str((path, arg, method_name)))
value_list.append(None)
kw[arg] = value_list
method = aq_base(method).__of__(self) # Use method in the context of portal_catalog
# Alter/Create row
try:
#start_time = DateTime()
method(**kw)
#end_time = DateTime()
#if method_name not in profile_dict:
# profile_dict[method_name] = end_time.timeTime() - start_time.timeTime()
#else:
# profile_dict[method_name] += end_time.timeTime() - start_time.timeTime()
#LOG('catalogObjectList', 0, '%s: %f seconds' % (method_name, profile_dict[method_name]))
except:
LOG("SQLCatalog Warning: could not catalog objects with method %s" % method_name,100, str(object_list))
raise
def uncatalogObject(self, path):
"""
Uncatalog and object from the Catalog.
......@@ -518,6 +1127,31 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
""" Accesses a single record for a given path """
return self.getMetadataForPath(path)
def getCatalogMethodIds(self):
"""Find Z SQL methods in the current folder and above
This function return a list of ids.
"""
ids={}
have_id=ids.has_key
StringType=type('')
while self is not None:
if hasattr(self, 'objectValues'):
for o in self.objectValues(valid_method_meta_type_list):
if hasattr(o,'id'):
id=o.id
if type(id) is not StringType: id=id()
if not have_id(id):
if hasattr(o,'title_and_id'): o=o.title_and_id()
else: o=id
ids[id]=id
if hasattr(self, 'aq_parent'): self=self.aq_parent
else: self=None
ids=map(lambda item: (item[1], item[0]), ids.items())
ids.sort()
return ids
def buildSQLQuery(self, query_table='catalog', REQUEST=None, **kw):
""" Builds a complex SQL query to simulate ZCalatog behaviour """
# Get search arguments:
......@@ -537,7 +1171,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
keyword_search_keys = self.sql_catalog_keyword_search_keys
topic_search_keys = self.sql_catalog_topic_search_keys
multivalue_keys = self.sql_catalog_multivalue_keys
# Define related maps
# each tuple has the form (key, 'table1,table2,table3/column/where_expression')
related_tuples = self.sql_catalog_related_keys
......@@ -548,7 +1182,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
related_column = {}
related_table_map = {}
table_rename_index = 0
related_methods = {} # related methods which need to be used
related_methods = {} # related methods which need to be used
for t in related_tuples:
t_tuple = t.split('|')
key = t_tuple[0].strip()
......@@ -562,20 +1196,20 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
if not related_table_map.has_key(method_id):
map_list = []
for table_id in join_tuple[0].split(','):
map_list.append((table_id,
map_list.append((table_id,
"related_%s_%s" % (table_id, table_rename_index))) # We add an index in order to alias tables in the join
table_rename_index += 1 # and prevent name conflicts
related_table_map[method_id] = map_list
related_table_map[method_id] = map_list
# We take additional parameters from the REQUEST
# and give priority to the REQUEST
if REQUEST is not None:
for key in acceptable_keys:
if REQUEST.has_key(key):
# Only copy a few keys from the REQUEST
if key in self.sql_catalog_request_keys:
if key in catalog_request_keys:
kw[key] = REQUEST[key]
# Let us try first not to use this
# Let us try first not to use this
#for key in related_keys:
# if REQUEST.has_key(key):
# kw[key] = REQUEST[key]
......@@ -590,14 +1224,14 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
for k, v in kw.items():
if k.endswith('_usage'):
new_k = k[0:-usage_len]
if not new_kw.has_key(new_k): new_kw[new_k] = {}
if not new_kw.has_key(new_k): new_kw[new_k] = {}
if type(new_kw[new_k]) is not type({}): new_kw[new_k] = {'query': new_kw[new_k]}
split_v = v.split(':')
split_v = v.split(':')
new_kw[new_k] = {split_v[0]: split_v[1]}
else:
if not new_kw.has_key(k):
new_kw[k] = v
else:
else:
new_kw[k]['query'] = v
kw = new_kw
#LOG('new kw', 0, str(kw))
......@@ -676,18 +1310,18 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
query_value = value['query']
if value.has_key('range'):
# This is a range
range_value = value['range']
range_value = value['range']
if range_value == 'min':
where_expression += ["%s >= '%s'" % (key, query_value)]
elif range_value == 'max':
where_expression += ["%s < '%s'" % (key, query_value)]
else:
where_expression += ["%s = %s" % (key, value)]
elif key in topic_search_keys:
# ERP5 CPS compatibility
elif key in topic_search_keys:
# ERP5 CPS compatibility
topic_operator = 'or'
if type(value) is type({}):
topic_operator = value.get('operator', 'or')
if type(value) is type({}):
topic_operator = value.get('operator', 'or')
value = value['query']
if type(value) is type(''):
topic_value = [value]
......@@ -704,14 +1338,14 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
elif query_table:
topic_key = query_table + '.' + topic_key
# Add table to table dict
from_table_dict[acceptable_key_map[topic_key][0]] = acceptable_key_map[topic_key][0] # We use catalog by default
from_table_dict[acceptable_key_map[topic_key][0]] = acceptable_key_map[topic_key][0] # We use catalog by default
query_item += ["%s = 1" % topic_key]
# Join
# Join
if len(query_item) > 0:
where_expression += ['(%s)' % join(query_item, ' %s ' % topic_operator)]
# Calculate extra where_expression based on required joins
for k, tid in from_table_dict.items():
if k != query_table:
if k != query_table:
where_expression.append('%s.uid = %s.uid' % (query_table, tid))
# Calculate extra where_expressions based on related definition
related_join_expression = []
......@@ -725,7 +1359,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
from_table_dict[t_tuple[1]] = t_tuple[0]
table_index += 1
where_expression.append(related_method(**table_id))
# Concatenate where_expressions
# Concatenate where_expressions
if kw.get('where_expression'):
if len(where_expression) > 0:
where_expression = "(%s) AND (%s)" % (kw['where_expression'], join(where_expression, ' AND ') )
......@@ -782,7 +1416,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
sort_on = str(sort_index)
except:
pass
# Use a dictionary at the moment.
return { 'from_table_list' : from_table_dict.items(),
'order_by_expression' : sort_on,
......@@ -808,15 +1442,13 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
# The used argument is deprecated and is ignored
try:
# Get the search method
#LOG("searchResults: kw:",0,str(kw))
method = getattr(self, self.sql_search_results)
# Return the result
kw['used'] = used
kw['REQUEST'] = REQUEST
return self.queryResults(method, **kw)
except:
LOG("Warning: could not search catalog",0,'', error=sys.exc_info())
LOG("Warning: could not search catalog",0,self.sql_search_results, error=sys.exc_info())
return []
__call__ = searchResults
......@@ -826,17 +1458,176 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
""" Returns the number of items which satisfy the where_expression """
try:
# Get the search method
#LOG("countResults: scr:",0,str(self.sql_count_results))
#LOG("countResults: used:",0,str(used))
#LOG("countResults: kw:",0,str(kw))
method = getattr(self, self.sql_count_results)
# Return the result
kw['used'] = used
kw['REQUEST'] = REQUEST
return self.queryResults(method, **kw)
except:
LOG("Warning: could not count catalog",0,str(self.sql_count_results), error=sys.exc_info())
LOG("Warning: could not count catalog",0,self.sql_count_results, error=sys.exc_info())
return [[0]]
def recordCatalogObject(self, path):
"""
Record the path of an object being catalogged.
"""
method = getattr(self, self.sql_record_catalog_object)
method(path=path)
def recordUncatalogObject(self, path):
"""
Record the path of an object being uncatalogged.
"""
method = getattr(self, self.sql_record_uncatalog_object)
method(path=path)
def deleteRecoredObjectList(self, path):
"""
Delete all objects which contain any path.
"""
if type(path) == type(''):
path = [path]
method = getattr(self, self.sql_delete_recorded_object_list)
method(path=path)
def readRecordedObjectList(self):
"""
Read objects. Note that this might not return all objects since ZMySQLDA limits the max rows.
"""
method = getattr(self, self.sql_read_recorded_object_list)
return method()
# Filtering
def manage_editFilter(self, REQUEST=None, RESPONSE=None, URL1=None):
"""
This methods allows to set a filter on each zsql method called,
so we can test if we should or not call a zsql method, so we can
increase a lot the speed.
"""
if withCMF:
for zsql_method in self.getFilterableMethodList():
# We will first look if the filter is activated
id = zsql_method.id
if not self.filter_dict.has_key(id):
self.filter_dict[id] = PersistentMapping()
self.filter_dict[id]['filtered']=0
self.filter_dict[id]['type']=[]
self.filter_dict[id]['expression']=""
if REQUEST.has_key('%s_box' % id):
self.filter_dict[id]['filtered'] = 1
else:
self.filter_dict[id]['filtered'] = 0
if REQUEST.has_key('%s_expression' % id):
expression = REQUEST['%s_expression' % id]
if expression == "":
self.filter_dict[id]['expression'] = ""
self.filter_dict[id]['expression_instance'] = None
else:
expr_instance = Expression(expression)
self.filter_dict[id]['expression'] = expression
self.filter_dict[id]['expression_instance'] = expr_instance
else:
self.filter_dict[id]['expression'] = ""
self.filter_dict[id]['expression_instance'] = None
if REQUEST.has_key('%s_type' % id):
list_type = REQUEST['%s_type' % id]
if type(list_type) is type('a'):
list_type = [list_type]
self.filter_dict[id]['type'] = list_type
else:
self.filter_dict[id]['type'] = []
if RESPONSE and URL1:
RESPONSE.redirect(URL1 + '/manage_catalogFilter?manage_tabs_message=Filter%20Changed')
def isMethodFiltered(self, method_name):
"""
Returns 1 if the method is already filtered,
else it returns 0
"""
if withCMF:
# Reset Filtet dict
# self.filter_dict= PersistentMapping()
if not hasattr(self,'filter_dict'):
self.filter_dict = PersistentMapping()
return 0
if self.filter_dict.has_key(method_name):
return self.filter_dict[method_name]['filtered']
return 0
def getExpression(self, method_name):
"""
Returns 1 if the method is already filtered,
else it returns 0
"""
if withCMF:
if not hasattr(self,'filter_dict'):
self.filter_dict = PersistentMapping()
return ""
if self.filter_dict.has_key(method_name):
return self.filter_dict[method_name]['expression']
return ""
def getExpressionInstance(self, method_name):
"""
Returns 1 if the method is already filtered,
else it returns 0
"""
if withCMF:
if not hasattr(self,'filter_dict'):
self.filter_dict = PersistentMapping()
return None
if self.filter_dict.has_key(method_name):
return self.filter_dict[method_name]['expression_instance']
return None
def isPortalTypeSelected(self, method_name,portal_type):
"""
Returns 1 if the method is already filtered,
else it returns 0
"""
if withCMF:
if not hasattr(self,'filter_dict'):
self.filter_dict = PersistentMapping()
return 0
if self.filter_dict.has_key(method_name):
result = portal_type in (self.filter_dict[method_name]['type'])
return result
return 0
def getFilterableMethodList(self):
"""
Returns only zsql methods wich catalog or uncatalog objets
"""
method_dict = {}
if withCMF:
methods = self.sql_catalog_object + self.sql_uncatalog_object \
+ self.sql_update_object + self.sql_catalog_object_list
for method_id in methods:
method_dict[method_id] = 1
method_list = map(lambda method_id: getattr(self, method_id, None), method_dict.keys())
return filter(lambda method: method is not None, method_list)
def getExpressionContext(self, ob):
'''
An expression context provides names for TALES expressions.
'''
if withCMF:
data = {
'here': ob,
'container': aq_parent(aq_inner(ob)),
'nothing': None,
'root': ob.getPhysicalRoot(),
'request': getattr( ob, 'REQUEST', None ),
'modules': SecureModuleImporter,
'user': getSecurityManager().getUser(),
}
return getEngine().getContext(data)
Globals.default__class_init__(Catalog)
class CatalogError(Exception): pass
product/ZSQLCatalog/ZSQLCatalog.py
View file @ cacdc675
......@@ -20,7 +20,7 @@ from OFS.Folder import Folder
from OFS.FindSupport import FindSupport
from OFS.ObjectManager import ObjectManager
from DateTime import DateTime
from Acquisition import Implicit
from Acquisition import Implicit, aq_base
from Persistence import Persistent
from DocumentTemplate.DT_Util import InstanceDict, TemplateDict
from DocumentTemplate.DT_Util import Eval
......@@ -28,15 +28,17 @@ from AccessControl.Permission import name_trans
from SQLCatalog import Catalog, CatalogError
from AccessControl import ClassSecurityInfo, getSecurityManager
from AccessControl.DTML import RestrictedDTML
import string, urllib, os, sys, time, types
import string, os, sys, types, threading
from zLOG import LOG
valid_method_meta_type_list = ('Z SQL Method','Script (Python)')
# Put the queue of catalogged objects in RAM for distributed computation.
catalogged_path_list = []
catalogged_path_list_lock = threading.Lock()
manage_addZCatalogForm=DTMLFile('dtml/addZCatalog',globals())
manage_addZSQLCatalogForm=DTMLFile('dtml/addZSQLCatalog',globals())
def manage_addZCatalog(self, id, title,
def manage_addZSQLCatalog(self, id, title,
vocab_id='create_default_catalog_',
REQUEST=None):
"""Add a ZCatalog object
......@@ -85,6 +87,8 @@ class ZCatalog(Folder, Persistent, Implicit):
'action': 'manage_catalogView',
'target': 'manage_main',
'help':('ZCatalog','ZCatalog_Cataloged-Objects.stx')},
{'label' : 'Filter', # TAB: Filter
'action' : 'manage_catalogFilter' },
{'label': 'Properties', # TAB: Properties
'action': 'manage_propertiesForm',
'help': ('OFSP','Properties.stx')},
......@@ -96,6 +100,9 @@ class ZCatalog(Folder, Persistent, Implicit):
'action': 'manage_catalogAdvanced',
'target':'manage_main',
'help':('ZCatalog','ZCatalog_Advanced.stx')},
{'label': 'Hot Reindexing', # TAB: Hot Reindex
'action': 'manage_catalogHotReindexing',
'target':'manage_main', },
{'label': 'Undo', # TAB: Undo
'action': 'manage_UndoForm',
'help': ('OFSP','Undo.stx')},
......@@ -114,12 +121,16 @@ class ZCatalog(Folder, Persistent, Implicit):
'catalog_object', 'uncatalog_object', 'refreshCatalog',
'manage_catalogView', 'manage_catalogFind',
'manage_catalogSchema',
'manage_catalogSchema', 'manage_catalogFilter',
'manage_catalogAdvanced', 'manage_objectInformation',
'manage_catalogHotReindexing',
'manage_catalogReindex', 'manage_catalogFoundItems', 'manage_catalogIndexAll',
'manage_catalogReindex', 'manage_catalogFoundItems',
'manage_catalogClear', 'manage_editSchema',
'manage_reindexIndex', 'manage_main',
'manage_editFilter',
'manage_hotReindexAll',
],
['Manager']),
......@@ -129,8 +140,13 @@ class ZCatalog(Folder, Persistent, Implicit):
'getpath', 'schema', 'names', 'columns', 'indexes', 'index_objects',
'all_meta_types', 'valid_roles', 'resolve_url',
'getobject', 'getObject', 'getObjectList', 'getCatalogSearchTableIds',
'getCatalogSearchResultKeys', ],
'getCatalogSearchResultKeys', 'getFilterableMethodList', ],
['Anonymous', 'Manager']),
('Import/Export objects',
['manage_catalogExportProperties', 'manage_catalogImportProperties', ],
['Manager']),
)
_properties = (
......@@ -138,147 +154,41 @@ class ZCatalog(Folder, Persistent, Implicit):
'description' : 'The title of this catalog',
'type' : 'string',
'mode' : 'w' },
{ 'id' : 'sql_catalog_produce_reserved',
'description' : 'A method to produce new uid values in advance',
{ 'id' : 'default_sql_catalog_id',
'description' : 'The id of the default SQL Catalog',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_clear_reserved',
'description' : 'A method to clear reserved uid values',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_object',
'description' : 'Methods to be called to catalog an object',
'type' : 'multiple selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_uncatalog_object',
'description' : 'Methods to be called to uncatalog an object',
'type' : 'multiple selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_update_object',
'description' : 'Methods will be called to updat a catalogued object',
'type' : 'multiple selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_clear_catalog',
'description' : 'The methods which should be called to clear the catalog',
'type' : 'multiple selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_search_results',
'description' : 'Main method to search the catalog',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_search_tables',
'description' : 'Tables to join in the result',
'type' : 'multiple selection',
'select_variable' : 'getTableIds',
'mode' : 'w' },
{ 'id' : 'sql_search_result_keys',
'description' : 'Keys to display in the result',
'type' : 'multiple selection',
'select_variable' : 'getResultColumnIds',
'mode' : 'w' },
{ 'id' : 'sql_count_results',
'description' : 'Main method to search the catalog',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_getitem_by_path',
'description' : 'Get a catalog brain by path',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_getitem_by_uid',
'description' : 'Get a catalog brain by uid',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_tables',
'description' : 'Method to get the main catalog tables',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_schema',
'description' : 'Method to get the main catalog schema',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_unique_values',
'description' : 'Find unique disctinct values in a column',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_paths',
'description' : 'List all object paths in catalog',
'type' : 'selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_keyword_search_keys',
'description' : 'Columns which should be considered as full text search',
'type' : 'multiple selection',
'select_variable' : 'getColumnIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_full_text_search_keys',
'description' : 'Columns which should be considered as full text search',
'type' : 'multiple selection',
'select_variable' : 'getColumnIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_request_keys',
'description' : 'Columns which should be ignore in the REQUEST in order to accelerate caching',
'type' : 'multiple selection',
'select_variable' : 'getColumnIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_multivalue_keys',
'description' : 'Keys which hold multiple values',
'type' : 'multiple selection',
'select_variable' : 'getColumnIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_topic_search_keys',
'description' : 'Columns which should be considered as topic index',
'type' : 'lines',
'select_variable' : 'getSQLCatalogIdList',
'mode' : 'w' },
{ 'id' : 'sql_catalog_related_keys',
'title' : 'Related keys',
'description' : 'Additional columns obtained through joins',
'type' : 'lines',
'mode' : 'w' },
)
sql_catalog_produce_reserved = 'z_produce_reserved_uid_list'
sql_catalog_clear_reserved = 'z_clear_reserved'
sql_catalog_object = ('catalog_object',)
sql_uncatalog_object = ('uncatalog_object',)
sql_update_object = ('update_object',)
sql_clear_catalog = ('drop_catalog', 'create_catalog')
sql_search_results = 'search_results'
sql_count_results = 'count_results'
sql_getitem_by_path = 'getitem_by_path'
sql_getitem_by_uid = 'getitem_by_uid'
sql_catalog_tables = 'catalog_tables'
sql_search_tables = ()
sql_catalog_schema = ('catalog_schema')
sql_unique_values = 'unique_values'
sql_catalog_paths = 'catalog_paths'
sql_catalog_keyword_search_keys = ('Description', 'Title', 'SearchableText')
sql_catalog_full_text_search_keys = ()
sql_catalog_request_keys = ()
sql_search_result_keys = ()
sql_catalog_topic_search_keys = ()
sql_catalog_multivalue_keys = ()
sql_catalog_related_keys = ()
# Hot Reindexing
{ 'id' : 'source_sql_catalog_id',
'description' : 'The id of a source SQL Catalog for hot reindexing',
'type' : 'string',
'mode' : '' },
{ 'id' : 'destination_sql_catalog_id',
'description' : 'The id of a destination SQL Catalog for hot reindexing',
'type' : 'string',
'mode' : '' },
{ 'id' : 'hot_reindexing_state',
'description' : 'The state of hot reindexing',
'type' : 'string',
'mode' : '' },
)
source_sql_catalog_id = None
destination_sql_catalog_id = None
hot_reindexing_state = None
default_sql_catalog_id = None
manage_catalogAddRowForm = DTMLFile('dtml/catalogAddRowForm', globals())
manage_catalogFilter = DTMLFile( 'dtml/catalogFilter', globals() )
manage_catalogView = DTMLFile('dtml/catalogView',globals())
manage_catalogFind = DTMLFile('dtml/catalogFind',globals())
manage_catalogSchema = DTMLFile('dtml/catalogSchema', globals())
manage_catalogIndexes = DTMLFile('dtml/catalogIndexes', globals())
manage_catalogAdvanced = DTMLFile('dtml/catalogAdvanced', globals())
manage_catalogHotReindexing = DTMLFile('dtml/catalogHotReindexing', globals())
manage_objectInformation = DTMLFile('dtml/catalogObjectInformation',
globals())
......@@ -287,27 +197,216 @@ class ZCatalog(Folder, Persistent, Implicit):
self=self.__of__(container)
self.id=id
self.title=title
self._catalog = Catalog()
def getSQLCatalogIdList(self):
return self.objectIds(spec=('SQLCatalog',))
def getSQLCatalog(self, id=None, default_value=None):
"""
Get the default SQL Catalog.
"""
if id is None:
if not self.default_sql_catalog_id:
id_list = self.getSQLCatalogIdList()
if len(id_list) > 0:
self.default_sql_catalog_id = id_list[0]
else:
return default_value
id = self.default_sql_catalog_id
return self._getOb(id)
def manage_catalogExportProperties(self, REQUEST=None, RESPONSE=None, sql_catalog_id=None):
"""
Export properties to an XML file.
"""
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return catalog.manage_exportProperties(REQUEST=REQUEST, RESPONSE=RESPONSE)
def manage_catalogImportProperties(self, file, sql_catalog_id=None):
"""
Import properties from an XML file.
"""
self.addColumn('id')
self.addIndex('id', 'FieldIndex')
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return catalog.manage_importProperties(file)
self.addColumn('title')
self.addIndex('title', 'TextIndex')
def __len__(self):
catalog = self.getSQLCatalog()
if catalog is None:
return 0
return len(catalog)
self.addColumn('meta_type')
self.addIndex('meta_type', 'FieldIndex')
def setHotReindexingState(self, state='', source_sql_catalog_id=None, destination_sql_catalog_id=None):
"""
Set the state of hot reindexing.
self.addColumn('bobobase_modification_time')
self.addIndex('bobobase_modification_time', 'FieldIndex')
Do not use setProperty because the state should not modified from the ZMI directly.
It must be maintained very carefully.
"""
if source_sql_catalog_id is None:
source_sql_catalog_id = self.default_sql_catalog_id
if state == 'finished':
self.hot_reindexing_state = None
self.source_sql_catalog_id = None
self.destination_sql_catalog_id = None
elif state == 'recording' or state == 'double indexing':
self.hot_reindexing_state = state
self.source_sql_catalog_id = source_sql_catalog_id
self.destination_sql_catalog_id = destination_sql_catalog_id
else:
raise CatalogError, 'unknown hot reindexing state %s' % state
self.addColumn('summary')
self.addIndex('PrincipiaSearchSource', 'TextIndex')
# This change must be reflected as soon as possible.
get_transaction().commit()
self.addIndex('path','PathIndex')
def playBackRecordedObjectList(self, sql_catalog_id=None):
"""
Play back must be a distinct method to activate...
"""
catalog = self.getSQLCatalog(sql_catalog_id)
while 1:
result = catalog.readRecordedObjectList()
if len(result) == 0:
break
get_transaction().commit()
path_dict = {}
for o in result:
if o.path not in path_dict:
path_dict[o.path] = None
object = self.resolve_path(o.path)
if o.catalog:
if object is not None:
self.activate(passive_commit=1, priority=5).reindexObject(object, sql_catalog_id=sql_catalog_id)
# Make sure that this object has no extra reference.
object = None
else:
if object is None:
self.uncatalog_object(o.path, sql_catalog_id=sql_catalog_id)
else:
# Make sure that this object has no extra reference.
object = None
get_transaction().commit()
catalog.deleteRecordedObjectList(path_dict.keys())
get_transaction().commit()
def exchangeDatabases(self, source_sql_catalog_id, destination_sql_catalog_id,
skin_selection_dict, sql_connection_id_dict):
"""
Exchange two databases.
"""
if self.default_sql_catalog_id == source_sql_catalog_id:
self.default_sql_catalog_id = destination_sql_catalog_id
for skin_name, selection in self.portal_skins.getSkinPaths():
if skin_name in skin_selection_dict:
new_selection = tuple(skin_selection_dict[skin_name])
self.portal_skins.manage_skinLayers(skinpath = new_selection, skinname = skin_name, add_skin = 1)
if sql_connection_id_dict:
def changeSQLConnectionIds(folder):
if folder.meta_type == 'Z SQL Method':
connection_id = folder.connection_id
if connection_id in sql_connection_id_dict:
folder.connection_id = sql_connection_id_dict[connection_id]
elif hasattr(aq_base(folder), 'objectValues'):
for object in folder.objectValues():
changeSQLConnectionIds(object)
changeSQLConnectionIds(self.portal_skins)
def manage_hotReindexAll(self, REQUEST, RESPONSE, URL1):
"""
Reindex objects from scratch in the background then switch to the newly created database.
"""
# Get parameters.
source_sql_catalog_id = REQUEST.get('source_sql_catalog_id')
destination_sql_catalog_id = REQUEST.get('destination_sql_catalog_id')
source_sql_connection_id_list = REQUEST.get('source_sql_connection_id_list')
destination_sql_connection_id_list = REQUEST.get('destination_sql_connection_id_list')
skin_name_list = REQUEST.get('skin_name_list')
skin_selection_list = REQUEST.get('skin_selection_list')
# Construct a mapping for skin selections.
skin_selection_dict = {}
for name, selection_list in zip(skin_name_list, skin_selection_list):
# Make sure that there is no extra space.
new_selection_list = []
for selection in selection_list:
new_selection = selection.strip()
if len(new_selection) > 0:
new_selection_list.append(new_selection)
skin_selection_dict[name] = new_selection_list
# Construct a mapping for connection ids.
sql_connection_id_dict = {}
for source_sql_connection_id, destination_sql_connection_id in zip(source_sql_connection_id_list, destination_sql_connection_id_list):
if source_sql_connection_id != destination_sql_connection_id:
sql_connection_id_dict[source_sql_connection_id] = destination_sql_connection_id
# Hot reindexing may not run for multiple databases.
if self.hot_reindexing_state is not None:
raise CatalogError, 'hot reindexing process is already running'
# Recreate the new database.
LOG('hotReindexObjectList', 0, 'Clearing the new database')
self.manage_catalogClear(sql_catalog_id=destination_sql_catalog_id)
def __len__(self): return len(self._catalog)
try:
# Start recording catalog/uncatalog information.
LOG('hotReindexObjectList', 0, 'Starting recording')
self.setHotReindexingState('recording',
source_sql_catalog_id=source_sql_catalog_id,
destination_sql_catalog_id=destination_sql_catalog_id)
# Iterate all objects recursively.
# XXX Commit transactions very often and use resolve_path to get objects instead of objectValues
# XXX This is not to be disturbed by normal user operations in the foreground.
# XXX Otherwise, many read conflicts happen and hot reindexing restarts again and again.
for path in self.Catalog_getIndexablePathList():
object = self.resolve_path(path)
if object is None: continue
id_list = object.objectIds()
object = None
get_transaction().commit() # Should not have references to objects too long.
LOG('hotReindexObjectList', 0, 'Catalogging %s' % path)
for id in id_list:
subpath = '/'.join([path, id])
o = self.resolve_path(subpath)
if o is not None:
o.activate(passive_commit=1, priority=5).recursiveQueueCataloggedObject(sql_catalog_id=destination_sql_catalog_id)
o = None
get_transaction().commit() # Should not have references to objects too long.
# Make sure that everything is catalogged.
LOG('hotReindexObjectList', 0, 'Flushing the queue')
self.activate(broadcast=1, passive_commit=1, after_method_id='immediateQueueCataloggedObject', priority=5).flushQueuedObjectList(sql_catalog_id=destination_sql_catalog_id)
# Now synchronize this new database with the current one.
# XXX It is necessary to use ActiveObject to wait for queued objects to be flushed.
LOG('hotReindexObjectList', 0, 'Starting double indexing')
self.activate(passive_commit=1, after_method_id='flushQueuedObjectList', priority=5).setHotReindexingState('double indexing',
source_sql_catalog_id=source_sql_catalog_id,
destination_sql_catalog_id=destination_sql_catalog_id)
# Play back records.
LOG('hotReindexObjectList', 0, 'Playing back records')
self.activate(passive_commit=1, after_method_id='setHotReindexingState', priority=5).playBackRecordedObjectList(sql_catalog_id=destination_sql_catalog_id)
# Exchange the databases.
LOG('hotReindexObjectList', 0, 'Exchanging databases')
self.activate(after_method_id=('reindexObject', 'playBackRecordedObjectList')).exchangeDatabases(source_sql_catalog_id, destination_sql_catalog_id, skin_selection_dict, sql_connection_id_dict)
finally:
# Finish.
LOG('hotReindexObjectList', 0, 'Finishing hot reindexing')
self.activate(passive_commit=1, after_method_id='exchageDatabases', priority=5).setHotReindexingState('finished')
RESPONSE.redirect(URL1 + '/manage_catalogHotReindexing?manage_tabs_message=Catalog%20Changed')
def manage_edit(self, RESPONSE, URL1, threshold=1000, REQUEST=None):
""" edit the catalog """
......@@ -318,135 +417,88 @@ class ZCatalog(Folder, Persistent, Implicit):
RESPONSE.redirect(URL1 + '/manage_main?manage_tabs_message=Catalog%20Changed')
def manage_catalogObject(self, REQUEST, RESPONSE, URL1, urls=None):
def manage_catalogObject(self, REQUEST, RESPONSE, URL1, urls=None, sql_catalog_id=None):
""" index Zope object(s) that 'urls' point to """
if urls:
if isinstance(urls, types.StringType):
urls=(urls,)
for url in urls:
obj = self.resolve_path(url)
if not obj:
obj = self.resolve_url(url, REQUEST)
if obj is not None:
self.catalog_object(obj, url)
if sql_catalog_id is None:
sql_catalog_id = REQUEST.get('sql_catalog_id', None)
RESPONSE.redirect(URL1 + '/manage_catalogView?manage_tabs_message=Object%20Cataloged')
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
catalog.manage_catalogObject(REQUEST, RESPONSE, URL1, urls=urls)
def manage_uncatalogObject(self, REQUEST, RESPONSE, URL1, urls=None):
def manage_uncatalogObject(self, REQUEST, RESPONSE, URL1, urls=None, sql_catalog_id=None):
""" removes Zope object(s) 'urls' from catalog """
if sql_catalog_id is None:
sql_catalog_id = REQUEST.get('sql_catalog_id', None)
if urls:
if isinstance(urls, types.StringType):
urls=(urls,)
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
catalog.manage_uncatalogObject(REQUEST, RESPONSE, URL1, urls=urls)
for url in urls:
self.uncatalog_object(url)
RESPONSE.redirect(URL1 + '/manage_catalogView?manage_tabs_message=Object%20Uncataloged')
def manage_catalogReindex(self, REQUEST, RESPONSE, URL1):
def manage_catalogReindex(self, REQUEST, RESPONSE, URL1, sql_catalog_id=None):
""" clear the catalog, then re-index everything """
elapse = time.time()
c_elapse = time.clock()
self.refreshCatalog(clear=1)
elapse = time.time() - elapse
c_elapse = time.clock() - c_elapse
if sql_catalog_id is None:
sql_catalog_id = REQUEST.get('sql_catalog_id', None)
RESPONSE.redirect(URL1 +
'/manage_catalogAdvanced?manage_tabs_message=' +
urllib.quote('Catalog Updated<br>'
'Total time: %s<br>'
'Total CPU time: %s' % (`elapse`, `c_elapse`)))
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
catalog.manage_catalogReindex(REQUEST, RESPONSE, URL1, urls=urls)
def refreshCatalog(self, clear=0):
def refreshCatalog(self, clear=0, sql_catalog_id=None):
""" re-index everything we can find """
cat = self._catalog
paths = cat.getPaths()
if clear:
cat.clear()
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
paths = catalog.getPaths()
if clear:
catalog.clear()
for p in paths:
obj = self.resolve_path(p.path)
if not obj:
obj = self.resolve_url(p.path, self.REQUEST)
if obj is not None:
self.catalog_object(obj, p.path)
def manage_catalogIndexAll(self, REQUEST, RESPONSE, URL1):
""" adds all objects to the catalog starting from the parent """
elapse = time.time()
c_elapse = time.clock()
def reindex(oself, r_dict):
path = oself.getPhysicalPath()
if r_dict.has_key(path): return
r_dict[path] = 1
try:
self.catalog_object(oself, '/'.join(path))
except:
# XXX better exception handling required
pass
for o in oself.objectValues():
reindex(o, r_dict)
new_dict = {}
reindex(self.aq_parent, new_dict)
elapse = time.time() - elapse
c_elapse = time.clock() - c_elapse
for p in paths:
obj = self.resolve_path(p.path)
if not obj:
obj = self.resolve_url(p.path, self.REQUEST)
if obj is not None:
self.catalog_object(obj, p.path, sql_catalog_id=sql_catalog_id)
RESPONSE.redirect(URL1 +
'/manage_catalogAdvanced?manage_tabs_message=' +
urllib.quote('Catalog Indexed<br>'
'Total time: %s<br>'
'Total CPU time: %s' % (`elapse`, `c_elapse`)))
def manage_catalogClear(self, REQUEST=None, RESPONSE=None, URL1=None):
def manage_catalogClear(self, REQUEST=None, RESPONSE=None, URL1=None, sql_catalog_id=None):
""" clears the whole enchilada """
self._catalog.clear()
if REQUEST and RESPONSE:
RESPONSE.redirect(URL1 + '/manage_catalogAdvanced?manage_tabs_message=Catalog%20Cleared')
if REQUEST is not None and sql_catalog_id is None:
sql_catalog_id = REQUEST.get('sql_catalog_id', None)
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
catalog.manage_catalogClear(REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL1)
def manage_catalogClearReserved(self, REQUEST=None, RESPONSE=None, URL1=None):
def manage_catalogClearReserved(self, REQUEST=None, RESPONSE=None, URL1=None, sql_catalog_id=None):
""" clears the whole enchilada """
self._catalog.clearReserved()
if REQUEST and RESPONSE:
RESPONSE.redirect(URL1 + '/manage_catalogAdvanced?manage_tabs_message=Catalog%20Cleared')
def manage_catalogCreateTables(self, REQUEST=None, RESPONSE=None, URL1=None):
""" creates the tables required for cataging objects """
self._catalog.createTables()
if REQUEST and RESPONSE:
RESPONSE.redirect(URL1 + '/manage_catalogAdvanced?manage_tabs_message=Tables%20Created')
if REQUEST is not None and sql_catalog_id is None:
sql_catalog_id = REQUEST.get('sql_catalog_id', None)
def manage_catalogDropTables(self, REQUEST=None, RESPONSE=None, URL1=None):
""" drops the tables required for cataging objects """
self._catalog.dropTables()
if REQUEST and RESPONSE:
RESPONSE.redirect(URL1 + '/manage_catalogAdvanced?manage_tabs_message=Tables%20Dropped')
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
catalog.manage_catalogClearReserved(REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL1)
def manage_catalogFoundItems(self, REQUEST, RESPONSE, URL2, URL1,
obj_metatypes=None,
obj_ids=None, obj_searchterm=None,
obj_expr=None, obj_mtime=None,
obj_mspec=None, obj_roles=None,
obj_permission=None):
obj_permission=None,
sql_catalog_id=None):
""" Find object according to search criteria and Catalog them
"""
if sql_catalog_id is None:
sql_catalog_id = REQUEST.get('sql_catalog_id', None)
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
catalog.manage_catalogFoundItems(REQUEST, RESPONSE, URL2, URL1,
obj_metatypes=obj_metatypes, obj_ids=obj_ids,
obj_searchterm=obj_searchterm, obj_expr=obj_expr,
obj_mtime=obj_mtime, obj_mspec=obj_mspec,
obj_roles=obj_roles, obj_permission=obj_permission)
elapse = time.time()
c_elapse = time.clock()
......@@ -468,7 +520,8 @@ class ZCatalog(Folder, Persistent, Implicit):
search_sub=1,
REQUEST=REQUEST,
apply_func=self.catalog_object,
apply_path=path)
apply_path=path,
sql_catalog_id=sql_catalog_id)
elapse = time.time() - elapse
c_elapse = time.clock() - c_elapse
......@@ -476,21 +529,33 @@ class ZCatalog(Folder, Persistent, Implicit):
RESPONSE.redirect(URL1 + '/manage_catalogView?manage_tabs_message=' +
urllib.quote('Catalog Updated<br>Total time: %s<br>Total CPU time: %s' % (`elapse`, `c_elapse`)))
def manage_editSchema(self, names, REQUEST=None, RESPONSE=None, URL1=None):
def manage_editSchema(self, names, REQUEST=None, RESPONSE=None, URL1=None, sql_catalog_id=None):
""" add a column """
self.editSchema(names)
if REQUEST is not None and sql_catalog_id is None:
sql_catalog_id = REQUEST.get('sql_catalog_id', None)
self.editSchema(names, sql_catalog_id=sql_catalog_id)
if REQUEST and RESPONSE:
RESPONSE.redirect(URL1 + '/manage_catalogSchema?manage_tabs_message=Schema%20Saved')
def newUid(self):
def newUid(self, sql_catalog_id=None):
"""
Allocates a new uid value.
"""
return self._catalog.newUid()
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return catalog.newUid()
def catalog_object(self, obj, uid=None, idxs=[], is_object_moved=0):
def wrapObject(self, object, **kw):
"""
Return a wrapped object for reindexing.
This method should be overridden if necessary.
"""
return object
def catalog_object(self, obj, uid=None, idxs=[], is_object_moved=0, sql_catalog_id=None):
""" wrapper around catalog """
if uid is None:
......@@ -504,46 +569,119 @@ class ZCatalog(Folder, Persistent, Implicit):
elif not isinstance(uid, types.StringType):
raise CatalogError('The object unique id must be a string.')
self._catalog.catalogObject(obj, uid, is_object_moved=is_object_moved)
obj = self.wrapObject(obj, sql_catalog_id=sql_catalog_id)
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
catalog.catalogObject(obj, uid, is_object_moved=is_object_moved)
if self.hot_reindexing_state is not None and self.source_sql_catalog_id == catalog.id:
destination_catalog = self.getSQLCatalog(self.destination_sql_catalog_id)
if self.hot_reindexing_state == 'recording':
destination_catalog.recordCatalogObject(uid)
else:
destination_catalog.deleteRecordedObjectList([uid]) # Prevent this object from being replayed.
destination_catalog.catalogObject(obj, uid, is_object_moved=is_object_moved,
sql_catalog_id=self.destination_sql_catalog_id)
security.declarePrivate('queueCataloggedObject')
def queueCataloggedObject(self, object, sql_catalog_id=None):
"""
Add an object into the queue for catalogging the object later in a batch form.
"""
catalogged_path_list_lock.acquire()
try:
catalogged_path_list.append(object.getPath())
size = len(catalogged_path_list)
finally:
catalogged_path_list_lock.release()
# It is better to flush the queued objects if they are too many...
if size > 100:
self.flushQueuedObjectList(sql_catalog_id=sql_catalog_id)
def uncatalog_object(self, uid):
security.declarePublic('flushQueuedObjectList')
def flushQueuedObjectList(self, sql_catalog_id=None):
"""
Flush queued objects.
"""
catalogged_path_list_lock.acquire()
try:
global catalogged_path_list
path_list = catalogged_path_list
catalogged_path_list = []
finally:
catalogged_path_list_lock.release()
object_list = []
for path in path_list:
object = self.resolve_path(path)
if object is not None:
object = self.wrapObject(object, sql_catalog_id=sql_catalog_id)
object_list.append(object)
if len(object_list) > 0:
catalog = self.getSQLCatalog(sql_catalog_id)
catalog.catalogObjectList(object_list)
def uncatalog_object(self, uid, sql_catalog_id=None):
""" wrapper around catalog """
self._catalog.uncatalogObject(uid)
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
catalog.uncatalogObject(uid)
if self.hot_reindexing_state is not None and self.source_sql_catalog_id == catalog.id:
destination_catalog = self.getSQLCatalog(self.destination_sql_catalog_id)
if self.hot_reindexing_state == 'recording':
destination_catalog.recordUncatalogObject(uid)
else:
destination_catalog.deleteRecordedObjectList([uid]) # Prevent this object from being replayed.
destination_catalog.uncatalogObject(uid)
def uniqueValuesFor(self, name):
def uniqueValuesFor(self, name, sql_catalog_id=None):
""" returns the unique values for a given FieldIndex """
return self._catalog.uniqueValuesFor(name)
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return catalog.uniqueValuesFor(name)
def getpath(self, uid):
return ()
def getpath(self, uid, sql_catalog_id=None):
"""
Return the path to a cataloged object given its uid
"""
object = self._catalog[uid]
if object:
return object.path
else:
return None
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
object = catalog[uid]
if object:
return object.path
else:
return None
getPath = getpath
def hasPath(self, path):
def hasPath(self, path, sql_catalog_id=None):
"""
Checks if path is catalogued
"""
return self._catalog.hasPath(path)
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return catalog.hasPath(path)
def getobject(self, uid, REQUEST=None):
def getobject(self, uid, REQUEST=None, sql_catalog_id=None):
"""
Return a cataloged object given its uid
"""
if REQUEST is not None and sql_catalog_id is None:
sql_catalog_id = REQUEST.get('sql_catalog_id', None)
try:
obj = self.aq_parent.unrestrictedTraverse(self.getpath(uid))
obj = self.aq_parent.unrestrictedTraverse(self.getpath(uid, sql_catalog_id=sql_catalog_id))
except:
LOG('WARNING: ZSQLCatalog',0,'Could not find object for uid %s' % uid)
obj = None
if obj is not None:
if REQUEST is None:
REQUEST=self.REQUEST
path = self.getpath(uid)
path = self.getpath(uid, sql_catalog_id=sql_catalog_id)
if path:
obj = self.resolve_url(path, REQUEST)
else:
......@@ -551,92 +689,117 @@ class ZCatalog(Folder, Persistent, Implicit):
return obj
getObject = getobject
def getObjectList(self, uid_list, REQUEST=None):
def getObjectList(self, uid_list, REQUEST=None, sql_catalog_id=None):
"""
Return a cataloged object given its uid
"""
obj_list = []
for uid in uid_list:
obj_list.append(self.getObject(uid, REQUEST))
obj_list.append(self.getObject(uid, REQUEST, sql_catalog_id=sql_catalog_id))
return obj_list
def getMetadataForUid(self, rid):
def getMetadataForUid(self, rid, sql_catalog_id=None):
"""return the correct metadata for the cataloged uid"""
return self._catalog.getMetadataForUid(int(rid))
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return catalog.getMetadataForUid(int(rid))
return {}
def getIndexDataForUid(self, rid):
def getIndexDataForUid(self, rid, sql_catalog_id=None):
"""return the current index contents for the specific uid"""
return self._catalog.getIndexDataForUid(rid)
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return catalog.getIndexDataForUid(rid)
return {}
# Aliases
getMetadataForRID = getMetadataForUid
getIndexDataForRID = getIndexDataForUid
def schema(self):
return self.getColumnIds()
def schema(self, sql_catalog_id=None):
return self.getColumnIds(sql_catalog_id=sql_catalog_id)
def indexes(self):
return self.getColumnIds()
def indexes(self, sql_catalog_id=None):
return self.getColumnIds(sql_catalog_id=sql_catalog_id)
def names(self):
return self._catalog.names
def names(self, sql_catalog_id=None):
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return catalog.names
return {}
def getColumnIds(self):
return self._catalog.getColumnIds()
def getColumnIds(self, sql_catalog_id=None):
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return catalog.getColumnIds()
return []
def getAttributesForColumn(self, column):
def getAttributesForColumn(self, column, sql_catalog_id=None):
"""
Return the attribute names as a single string
"""
return string.join(self.names().get(column, ('',)),' ')
return string.join(self.names(sql_catalog_id=sql_catalog_id).get(column, ('',)),' ')
def _searchable_arguments(self):
return self._catalog.getColumnIds()
def _searchable_arguments(self, sql_catalog_id=None):
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return catalog.getColumnIds(sql_catalog_id=sql_catalog_id)
return []
def editSchema(self,names):
self._catalog.editSchema(names)
def editSchema(self,names, sql_catalog_id=None):
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
catalog.editSchema(names)
def _searchable_result_columns(self):
def _searchable_result_columns(self, sql_catalog_id=None):
r = []
for name in self._catalog.getColumnIds():
i = {}
i['name'] = name
i['type'] = 's'
i['parser'] = str
i['width'] = 8
r.append(i)
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
for name in catalog.getColumnIds():
i = {}
i['name'] = name
i['type'] = 's'
i['parser'] = str
i['width'] = 8
r.append(i)
r.append({'name': 'data_record_id_',
'type': 's',
'parser': str,
'width': 8})
return r
def getColumnIds(self):
return self._catalog.getColumnIds()
security.declarePublic('buildSQLQuery')
def buildSQLQuery(self, REQUEST=None, query_table='catalog', **kw):
def buildSQLQuery(self, REQUEST=None, query_table='catalog', sql_catalog_id=None, **kw):
"""
Build a SQL query from keywords.
If query_table is specified, it is used as the table name instead of 'catalog'.
"""
return self._catalog.buildSQLQuery(REQUEST=REQUEST, query_table=query_table, **kw)
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return catalog.buildSQLQuery(REQUEST=REQUEST, query_table=query_table, **kw)
return ''
def searchResults(self, REQUEST=None, used=None, **kw):
def searchResults(self, REQUEST=None, used=None, sql_catalog_id=None, **kw):
"""
Search the catalog according to the ZTables search interface.
Search terms can be passed in the REQUEST or as keyword
arguments.
"""
return apply(self._catalog.searchResults, (REQUEST,used), kw)
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return apply(catalog.searchResults, (REQUEST,used), kw)
return []
__call__=searchResults
def countResults(self, REQUEST=None, used=None, **kw):
def countResults(self, REQUEST=None, used=None, sql_catalog_id=None, **kw):
"""
Counts the number of items which satisfy the query defined in kw.
"""
return apply(self._catalog.countResults, (REQUEST,used), kw)
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return apply(catalog.countResults, (REQUEST,used), kw)
return []
## this stuff is so the find machinery works
......@@ -668,7 +831,8 @@ class ZCatalog(Folder, Persistent, Implicit):
obj_permission=None, obj_roles=None,
search_sub=0,
REQUEST=None, result=None, pre='',
apply_func=None, apply_path=''):
apply_func=None, apply_path='',
sql_catalog_id=None):
"""Zope Find interface and apply
This is a *great* hack. Zope find just doesn't do what we
......@@ -743,7 +907,7 @@ class ZCatalog(Folder, Persistent, Implicit):
)
):
if apply_func:
apply_func(ob, (apply_path+'/'+p))
apply_func(ob, (apply_path+'/'+p), sql_catalog_id=sql_catalog_id)
else:
add_result((p, ob))
dflag=0
......@@ -755,7 +919,8 @@ class ZCatalog(Folder, Persistent, Implicit):
obj_permission, obj_roles,
search_sub,
REQUEST, result, p,
apply_func, apply_path)
apply_func, apply_path,
sql_catalog_id)
if dflag: ob._p_deactivate()
return result
......@@ -783,102 +948,116 @@ class ZCatalog(Folder, Persistent, Implicit):
try: return self.unrestrictedTraverse(path)
except: pass
def manage_normalize_paths(self, REQUEST):
def manage_normalize_paths(self, REQUEST, sql_catalog_id=None):
"""Ensure that all catalog paths are full physical paths
This should only be used with ZCatalogs in which all paths can
be resolved with unrestrictedTraverse."""
paths = self._catalog.paths
uids = self._catalog.uids
unchanged = 0
fixed = []
removed = []
for path, rid in uids.items():
ob = None
if path[:1] == '/':
ob = self.resolve_url(path[1:],REQUEST)
if ob is None:
ob = self.resolve_url(path, REQUEST)
if sql_catalog_id is None:
sql_catalog_id = REQUEST.get('sql_catalog_id', None)
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
paths = catalog.paths
uids = catalog.uids
unchanged = 0
fixed = []
removed = []
for path, rid in uids.items():
ob = None
if path[:1] == '/':
ob = self.resolve_url(path[1:],REQUEST)
if ob is None:
removed.append(path)
continue
ppath = string.join(ob.getPhysicalPath(), '/')
if path != ppath:
fixed.append((path, ppath))
else:
unchanged = unchanged + 1
ob = self.resolve_url(path, REQUEST)
if ob is None:
removed.append(path)
continue
ppath = string.join(ob.getPhysicalPath(), '/')
if path != ppath:
fixed.append((path, ppath))
else:
unchanged = unchanged + 1
for path, ppath in fixed:
rid = uids[path]
del uids[path]
paths[rid] = ppath
uids[ppath] = rid
for path in removed:
self.uncatalog_object(path)
for path, ppath in fixed:
rid = uids[path]
del uids[path]
paths[rid] = ppath
uids[ppath] = rid
for path in removed:
self.uncatalog_object(path, sql_catalog_id=sql_catalog_id)
return MessageDialog(title='Done Normalizing Paths',
message='%s paths normalized, %s paths removed, and '
'%s unchanged.' % (len(fixed), len(removed), unchanged),
action='./manage_main')
def getTableIds(self):
def getTableIds(self, sql_catalog_id=None):
"""Returns all tables of this catalog
"""
return self._catalog.getTableIds()
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return catalog.getTableIds()
return []
def getCatalogSearchResultKeys(self):
def getCatalogSearchResultKeys(self, sql_catalog_id=None):
"""Return selected tables of catalog which are used in JOIN.
catalaog is always first
"""
return self.sql_search_result_keys
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return catalog.sql_search_result_keys
return []
def getCatalogSearchTableIds(self):
def getCatalogSearchTableIds(self, sql_catalog_id=None):
"""Return selected tables of catalog which are used in JOIN.
catalaog is always first
"""
if len(self.sql_search_tables) > 0:
if self.sql_search_tables[0] != 'catalog':
result = ['catalog']
for t in self.sql_search_tables:
if t != 'catalog':
result.append(t)
self.sql_search_tables = result
else:
self.sql_search_tables = ['catalog']
return self.sql_search_tables
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return catalog.getCatalogSearchTableIds()
return []
def getResultColumnIds(self):
def getResultColumnIds(self, sql_catalog_id=None):
"""Return selected tables of catalog which are used
as metadata
"""
return self._catalog.getResultColumnIds()
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return catalog.getResultColumnIds()
return []
def getCatalogMethodIds(self):
def getCatalogMethodIds(self, sql_catalog_id=None):
"""Find Z SQL methods in the current folder and above
This function return a list of ids.
"""
ids={}
have_id=ids.has_key
StringType=type('')
while self is not None:
if hasattr(self, 'objectValues'):
for o in self.objectValues(valid_method_meta_type_list):
if hasattr(o,'id'):
id=o.id
if type(id) is not StringType: id=id()
if not have_id(id):
if hasattr(o,'title_and_id'): o=o.title_and_id()
else: o=id
ids[id]=id
if hasattr(self, 'aq_parent'): self=self.aq_parent
else: self=None
ids=map(lambda item: (item[1], item[0]), ids.items())
ids.sort()
return ids
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return catalog.getCatalogMethodIds()
return {}
def manage_editFilter(self, REQUEST=None, RESPONSE=None, URL1=None, sql_catalog_id=None):
"""
This methods allows to set a filter on each zsql method called,
so we can test if we should or not call a zsql method, so we can
increase a lot the speed.
"""
if REQUEST is not None and sql_catalog_id is None:
sql_catalog_id = REQUEST.get('sql_catalog_id', None)
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
catalog.manage_editFilter(REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL1)
def getFilterableMethodList(self, sql_catalog_id=None):
"""
Returns only zsql methods wich catalog or uncatalog objets
"""
catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None:
return catalog.getFilterableMethodList()
return []
Globals.default__class_init__(ZCatalog)
......
product/ZSQLCatalog/__init__.py
View file @ cacdc675
......@@ -17,14 +17,24 @@ import ZSQLCatalog, SQLCatalog
from ZClasses import createZClassForBase
createZClassForBase( ZSQLCatalog.ZCatalog , globals()
, 'ZCatalogBase', 'ZCatalog' )
, 'ZSQLCatalogBase', 'ZSQLCatalog' )
createZClassForBase( SQLCatalog.Catalog , globals()
, 'SQLCatalogBase', 'SQLCatalog' )
def initialize(context):
context.registerClass(
ZSQLCatalog.ZCatalog,
permission='Add ZCatalogs',
constructors=(ZSQLCatalog.manage_addZCatalogForm,
ZSQLCatalog.manage_addZCatalog),
constructors=(ZSQLCatalog.manage_addZSQLCatalogForm,
ZSQLCatalog.manage_addZSQLCatalog),
icon='www/ZCatalog.gif',
)
context.registerClass(
SQLCatalog.Catalog,
permission='Add ZCatalogs',
constructors=(SQLCatalog.manage_addSQLCatalogForm,
SQLCatalog.manage_addSQLCatalog),
icon='www/ZCatalog.gif',
)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7