Commit fe8a3f4d authored by Hanno Schlichting's avatar Hanno Schlichting

Make the report query key calculations saner, by sanitizing the query and...

Make the report query key calculations saner, by sanitizing the query and actually using it to execute the queries. Otherwise there's too much potential for differences in the key calculation and the actual index search code.
parent 3c8ca26f
...@@ -35,6 +35,13 @@ Bugs Fixed ...@@ -35,6 +35,13 @@ Bugs Fixed
Restructuring Restructuring
+++++++++++++ +++++++++++++
- Cleaned up the Products.ZCatalog search API's. The deprecated support for
using `<index id>_usage` arguments in the request has been removed. Support
for overriding operators via the `<index id>_operator` syntax has been
limited to the query value for each index and no longer works directly on
the request. The query is now brought into a canonical form before being
passed into the `_apply_index` method of each index.
- Factored out the `Products.MailHost` package into its own distributions. It - Factored out the `Products.MailHost` package into its own distributions. It
will no longer be included by default in Zope 2.14 but live on as an will no longer be included by default in Zope 2.14 but live on as an
independent add-on. independent add-on.
......
...@@ -250,10 +250,6 @@ class DateRangeIndex(UnIndex): ...@@ -250,10 +250,6 @@ class DateRangeIndex(UnIndex):
If the request does not contain the needed parameters, then If the request does not contain the needed parameters, then
return None. return None.
If the request contains a parameter with the name of the
column + "_usage", snif for information on how to handle
applying the index.
Otherwise return two objects. The first object is a ResultSet Otherwise return two objects. The first object is a ResultSet
containing the record numbers of the matching records. The containing the record numbers of the matching records. The
second object is a tuple containing the names of all data fields second object is a tuple containing the names of all data fields
......
...@@ -66,7 +66,7 @@ Changes to ZCatalog ...@@ -66,7 +66,7 @@ Changes to ZCatalog
- Method 2: The query and all parameters to be passed to an index XXX are - Method 2: The query and all parameters to be passed to an index XXX are
passed as dictionary inside the request dictionary. Example: passed as dictionary inside the request dictionary. Example:
old: <dtml-in myCatalog(myindex='xx yy',myindex_usage':'blabla') old: <dtml-in myCatalog(myindex='xx yy')
new: <dtml-in myCatalog(myindex={'query':'xx yy','XXXXX':'blabla') new: <dtml-in myCatalog(myindex={'query':'xx yy','XXXXX':'blabla')
Please check the indexes documentation for informations about additional Please check the indexes documentation for informations about additional
......
...@@ -322,12 +322,8 @@ class UnIndex(SimpleItem): ...@@ -322,12 +322,8 @@ class UnIndex(SimpleItem):
- if the value is a sequence, return a union search. - if the value is a sequence, return a union search.
If the request contains a parameter with the name of the - If the value is a dict and contains a key of the form
column + '_usage', it is sniffed for information on how to '<index>_operator' this overrides the default method
handle applying the index.
If the request contains a parameter with the name of the
column = '_operator' this overrides the default method
('or') to combine search results. Valid values are "or" ('or') to combine search results. Valid values are "or"
and "and". and "and".
......
...@@ -11,17 +11,18 @@ ...@@ -11,17 +11,18 @@
# #
############################################################################## ##############################################################################
"""PluginIndexes utils. """PluginIndexes utils.
$Id$
""" """
from types import InstanceType from types import InstanceType
from warnings import warn
from DateTime.DateTime import DateTime from DateTime.DateTime import DateTime
class parseIndexRequest: class IndexRequestParseError(Exception):
pass
class parseIndexRequest:
""" """
This class provides functionality to hide the internals of a request This class provides functionality to hide the internals of a request
send from the Catalog/ZCatalog to an index._apply_index() method. send from the Catalog/ZCatalog to an index._apply_index() method.
...@@ -30,7 +31,6 @@ class parseIndexRequest: ...@@ -30,7 +31,6 @@ class parseIndexRequest:
- old-style parameters where the query for an index as value inside - old-style parameters where the query for an index as value inside
the request directory where the index name is the name of the key. the request directory where the index name is the name of the key.
Additional parameters for an index could be passed as index+"_usage" ...
- dictionary-style parameters specify a query for an index as - dictionary-style parameters specify a query for an index as
an entry in the request dictionary where the key corresponds to the an entry in the request dictionary where the key corresponds to the
...@@ -50,7 +50,7 @@ class parseIndexRequest: ...@@ -50,7 +50,7 @@ class parseIndexRequest:
parameters parameters
""" """
ParserException = 'IndexRequestParseError' ParserException = IndexRequestParseError
def __init__(self, request, iid, options=[]): def __init__(self, request, iid, options=[]):
""" parse a request from the ZPublisher and return a uniform """ parse a request from the ZPublisher and return a uniform
...@@ -66,16 +66,6 @@ class parseIndexRequest: ...@@ -66,16 +66,6 @@ class parseIndexRequest:
self.keys = None self.keys = None
return return
# We keep this for backward compatility
usage_param = iid + '_usage'
if request.has_key(usage_param):
self.usage = request[usage_param]
warn("ZCatalog query using '%s' detected.\n"
"Using query parameters ending with '_usage' is deprecated.\n"
"Consider using record-style parameters instead "
"(see lib/python/Products/PluginIndexes/README.txt for "
"details)" % usage_param, DeprecationWarning)
param = request[iid] param = request[iid]
keys = None keys = None
...@@ -85,7 +75,7 @@ class parseIndexRequest: ...@@ -85,7 +75,7 @@ class parseIndexRequest:
record = param record = param
if not hasattr(record, 'query'): if not hasattr(record, 'query'):
raise self.ParserException, ( raise self.ParserException(
"record for '%s' *must* contain a " "record for '%s' *must* contain a "
"'query' attribute" % self.id) "'query' attribute" % self.id)
keys = record.query keys = record.query
......
...@@ -63,10 +63,6 @@ class IPluggableIndex(Interface): ...@@ -63,10 +63,6 @@ class IPluggableIndex(Interface):
If the request does not contain the needed parameters, then If the request does not contain the needed parameters, then
None is returned. None is returned.
If the request contains a parameter with the name of the column
+ "_usage", it is sniffed for information on how to handle applying
the index. (Note: this style or parameters is deprecated)
If the request contains a parameter with the name of the If the request contains a parameter with the name of the
column and this parameter is either a Record or a class column and this parameter is either a Record or a class
instance then it is assumed that the parameters of this index instance then it is assumed that the parameters of this index
......
...@@ -440,7 +440,40 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -440,7 +440,40 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
## This is the Catalog search engine. Most of the heavy lifting happens below ## This is the Catalog search engine. Most of the heavy lifting happens below
def search(self, request, sort_index=None, reverse=0, limit=None, merge=1): def make_query(self, request):
# This is a bit of a mess, but the ZCatalog API has traditionally
# supported passing in query restrictions in almost arbitary ways
real_req = None
if isinstance(request, dict):
query = request.copy()
elif isinstance(request, CatalogSearchArgumentsMap):
query = {}
query.update(request.keywords)
real_req = request.request
if isinstance(request.request, dict):
query.update(real_req)
else:
real_req = request.request
else:
real_req = request
if real_req is not None:
# TODO: This deserves depreaction
known_keys = query.keys()
# The request has too many places where an index restriction
# might be specified. Putting all of request.form,
# request.other, ... into the query isn't what we want.
# So we iterate over all known indexes instead and see if they
# are in the request.
for iid in self.indexes.keys():
if iid in known_keys:
continue
value = real_req.get(iid)
if value:
query[iid] = value
return query
def search(self, query, sort_index=None, reverse=0, limit=None, merge=1):
"""Iterate through the indexes, applying the query to each one. If """Iterate through the indexes, applying the query to each one. If
merge is true then return a lazy result set (sorted if appropriate) merge is true then return a lazy result set (sorted if appropriate)
otherwise return the raw (possibly scored) results for later merging. otherwise return the raw (possibly scored) results for later merging.
...@@ -453,13 +486,16 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -453,13 +486,16 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
rs = None # resultset rs = None # resultset
# Indexes fulfill a fairly large contract here. We hand each # Indexes fulfill a fairly large contract here. We hand each
# index the request mapping we are given (which may be composed # index the query mapping we are given (which may be composed
# of some combination of web request, kw mappings or plain old dicts) # of some combination of web request, kw mappings or plain old dicts)
# and the index decides what to do with it. If the index finds work # and the index decides what to do with it. If the index finds work
# for itself in the request, it returns the results and a tuple of # for itself in the query, it returns the results and a tuple of
# the attributes that were used. If the index finds nothing for it # the attributes that were used. If the index finds nothing for it
# to do then it returns None. # to do then it returns None.
# Canonicalize the request into a sensible query before passing it on
query = self.make_query(query)
# For hysterical reasons, if all indexes return None for a given # For hysterical reasons, if all indexes return None for a given
# request (and no attributes were used) then we append all results # request (and no attributes were used) then we append all results
# in the Catalog. This generally happens when the search values # in the Catalog. This generally happens when the search values
...@@ -469,7 +505,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -469,7 +505,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
# Note that if the indexes find query arguments, but the end result # Note that if the indexes find query arguments, but the end result
# is an empty sequence, we do nothing # is an empty sequence, we do nothing
cr = self.getCatalogReport(request) cr = self.getCatalogReport(query)
cr.start() cr.start()
for i in self.indexes.keys(): for i in self.indexes.keys():
...@@ -479,7 +515,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -479,7 +515,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
continue continue
cr.split(i) cr.split(i)
r = _apply_index(request) r = _apply_index(query)
cr.split(i) cr.split(i)
if r is not None: if r is not None:
...@@ -491,7 +527,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -491,7 +527,7 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
cr.stop() cr.stop()
if rs is None: if rs is None:
# None of the indexes found anything to do with the request # None of the indexes found anything to do with the query
# We take this to mean that the query was empty (an empty filter) # We take this to mean that the query was empty (an empty filter)
# and so we return everything in the catalog # and so we return everything in the catalog
if sort_index is None: if sort_index is None:
...@@ -729,9 +765,12 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -729,9 +765,12 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
return None return None
def searchResults(self, REQUEST=None, used=None, _merge=1, **kw): def searchResults(self, REQUEST=None, used=None, _merge=1, **kw):
# You should pass in a simple dictionary as the request argument,
# which only contains the relevant query.
# The used argument is deprecated and is ignored # The used argument is deprecated and is ignored
if REQUEST is None and not kw: if REQUEST is None and not kw:
# Try to acquire request if we get no args for bw compat # Try to acquire request if we get no args for bw compat
# TODO: Should be deprecated
REQUEST = getattr(self, 'REQUEST', None) REQUEST = getattr(self, 'REQUEST', None)
args = CatalogSearchArgumentsMap(REQUEST, kw) args = CatalogSearchArgumentsMap(REQUEST, kw)
sort_index = self._getSortIndex(args) sort_index = self._getSortIndex(args)
...@@ -747,12 +786,12 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -747,12 +786,12 @@ class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
__call__ = searchResults __call__ = searchResults
def getCatalogReport(self, request=None): def getCatalogReport(self, query=None):
"""Reports about the duration of queries. """Reports about the duration of queries.
""" """
parent = Acquisition.aq_base(Acquisition.aq_parent(self)) parent = Acquisition.aq_base(Acquisition.aq_parent(self))
threshold = getattr(parent, 'long_query_time', 0.1) threshold = getattr(parent, 'long_query_time', 0.1)
return CatalogReport(self, request, threshold) return CatalogReport(self, query, threshold)
class CatalogSearchArgumentsMap: class CatalogSearchArgumentsMap:
......
...@@ -75,37 +75,9 @@ addCleanUp(clear_value_indexes) ...@@ -75,37 +75,9 @@ addCleanUp(clear_value_indexes)
del addCleanUp del addCleanUp
def make_query(indexes, request): def make_key(catalog, query):
# This is a bit of a mess, but the ZCatalog API supports passing
# in query restrictions in almost arbitary ways
if isinstance(request, dict):
query = request.copy()
else:
query = {}
query.update(request.keywords)
real_req = request.request
if isinstance(real_req, dict):
query.update(real_req)
known_keys = query.keys()
# The request has too many places where an index restriction might be
# specified. Putting all of request.form, request.other, ... into the
# key isn't what we want either, so we iterate over all known indexes
# instead and see if they are in the request.
for iid in indexes.keys():
if iid in known_keys:
continue
value = real_req.get(iid)
if value:
query[iid] = value
return query
def make_key(catalog, request):
indexes = catalog.indexes indexes = catalog.indexes
valueindexes = determine_value_indexes(indexes) valueindexes = determine_value_indexes(indexes)
query = make_query(indexes, request)
key = keys = query.keys() key = keys = query.keys()
values = [name for name in keys if name in valueindexes] values = [name for name in keys if name in valueindexes]
...@@ -167,11 +139,11 @@ class CatalogReport(StopWatch): ...@@ -167,11 +139,11 @@ class CatalogReport(StopWatch):
"""Catalog report class to meassure and identify catalog queries. """Catalog report class to meassure and identify catalog queries.
""" """
def __init__(self, catalog, request=None, threshold=0.1): def __init__(self, catalog, query=None, threshold=0.1):
super(CatalogReport, self).__init__() super(CatalogReport, self).__init__()
self.catalog = catalog self.catalog = catalog
self.request = request self.query = query
self.threshold = threshold self.threshold = threshold
parent = aq_parent(catalog) parent = aq_parent(catalog)
...@@ -195,7 +167,7 @@ class CatalogReport(StopWatch): ...@@ -195,7 +167,7 @@ class CatalogReport(StopWatch):
# The key calculation takes a bit itself, we want to avoid that for # The key calculation takes a bit itself, we want to avoid that for
# any fast queries. This does mean that slow queries get the key # any fast queries. This does mean that slow queries get the key
# calculation overhead added to their runtime. # calculation overhead added to their runtime.
key = make_key(self.catalog, self.request) key = make_key(self.catalog, self.query)
reports_lock.acquire() reports_lock.acquire()
try: try:
......
...@@ -389,9 +389,11 @@ class TestCatalogObject(unittest.TestCase): ...@@ -389,9 +389,11 @@ class TestCatalogObject(unittest.TestCase):
a = self._catalog({}) a = self._catalog({})
self.assertEqual(len(a), upper, self.assertEqual(len(a), upper,
'length should be %s, its %s' % (upper, len(a))) 'length should be %s, its %s' % (upper, len(a)))
# Queries consisting of empty strings should do the same # Queries used to do the same, because of a bug in the
# parseIndexRequest function, mistaking a CatalogSearchArgumentsMap
# for a Record class
a = self._catalog({'col1':'', 'col2':'', 'col3':''}) a = self._catalog({'col1':'', 'col2':'', 'col3':''})
self.assertEqual(len(a), upper, self.assertEqual(len(a), 0,
'length should be %s, its %s' % (upper, len(a))) 'length should be %s, its %s' % (upper, len(a)))
def testFieldIndexLength(self): def testFieldIndexLength(self):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment