Skip to content

Z
Zope
Commit 97bb6b12 authored by Hanno Schlichting's avatar Hanno Schlichting
Browse files
Options

Factored out the `Products.ZCatalog` and `Products.PluginIndexes` packages... 

Factored out the `Products.ZCatalog` and `Products.PluginIndexes` packages into a new `Products.ZCatalog` distribution.
parent 03a5df08
Show whitespace changes
Inline Side-by-side
Showing with 10 additions and 12182 deletions
buildout.cfg
View file @ 97bb6b12
...@@ -59,6 +59,7 @@ eggs = ...@@ -59,6 +59,7 @@ eggs =
Products.OFSP Products.OFSP
Products.PythonScripts Products.PythonScripts
Products.StandardCacheManagers Products.StandardCacheManagers
Products.ZCatalog
Products.ZCTextIndex Products.ZCTextIndex
Record Record
RestrictedPython RestrictedPython
......
doc/CHANGES.rst
View file @ 97bb6b12
...@@ -11,10 +11,14 @@ http://docs.zope.org/zope2/releases/. ...@@ -11,10 +11,14 @@ http://docs.zope.org/zope2/releases/.
Bugs Fixed Bugs Fixed
++++++++++ ++++++++++
- Fix `LazyMap` to avoid unnecessary function calls.
- LP 686664: WebDAV Lock Manager ZMI view wasn't accessible. - LP 686664: WebDAV Lock Manager ZMI view wasn't accessible.
Restructuring
+++++++++++++
- Factored out the `Products.ZCatalog` and `Products.PluginIndexes` packages
into a new `Products.ZCatalog` distribution.
2.13.1 (2010-12-07) 2.13.1 (2010-12-07)
------------------- -------------------
......
setup.py
View file @ 97bb6b12
...@@ -54,6 +54,7 @@ setup(name='Zope2', ...@@ -54,6 +54,7 @@ setup(name='Zope2',
'MultiMapping', 'MultiMapping',
'Persistence', 'Persistence',
'Products.OFSP >= 2.13.2', 'Products.OFSP >= 2.13.2',
'Products.ZCatalog',
'Products.ZCTextIndex', 'Products.ZCTextIndex',
'Record', 'Record',
'RestrictedPython', 'RestrictedPython',
......
sources.cfg
View file @ 97bb6b12
...@@ -15,6 +15,7 @@ Products.MIMETools = svn ^/Products.MIMETools/trunk ...@@ -15,6 +15,7 @@ Products.MIMETools = svn ^/Products.MIMETools/trunk
Products.OFSP = svn ^/Products.OFSP/trunk Products.OFSP = svn ^/Products.OFSP/trunk
Products.PythonScripts = svn ^/Products.PythonScripts/trunk Products.PythonScripts = svn ^/Products.PythonScripts/trunk
Products.StandardCacheManagers = svn ^/Products.StandardCacheManagers/trunk Products.StandardCacheManagers = svn ^/Products.StandardCacheManagers/trunk
Products.ZCatalog = svn ^/Products.ZCatalog/trunk
Products.ZCTextIndex = svn ^/Products.ZCTextIndex/trunk Products.ZCTextIndex = svn ^/Products.ZCTextIndex/trunk
Record = svn ^/Record/trunk Record = svn ^/Record/trunk
tempstorage = svn ^/tempstorage/trunk tempstorage = svn ^/tempstorage/trunk
......
src/Products/PluginIndexes/BooleanIndex/BooleanIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
from logging import getLogger
from App.special_dtml import DTMLFile
from BTrees.IIBTree import IIBTree, IITreeSet, IISet
from BTrees.IIBTree import union, intersection, difference
import BTrees.Length
from ZODB.POSException import ConflictError
from Products.PluginIndexes.common.util import parseIndexRequest
from Products.PluginIndexes.common.UnIndex import UnIndex
_marker = object()
LOG = getLogger('BooleanIndex.UnIndex')
class BooleanIndex(UnIndex):
"""Index for booleans
self._index = set([documentId1, documentId2])
self._unindex = {documentId:[True/False]}
False doesn't have actual entries in _index.
"""
meta_type = "BooleanIndex"
manage_options= (
{'label': 'Settings',
'action': 'manage_main'},
{'label': 'Browse',
'action': 'manage_browse'},
)
query_options = ["query"]
manage = manage_main = DTMLFile('dtml/manageBooleanIndex', globals())
manage_main._setName('manage_main')
manage_browse = DTMLFile('../dtml/browseIndex', globals())
def clear(self):
self._length = BTrees.Length.Length()
self._index = IITreeSet()
self._unindex = IIBTree()
def insertForwardIndexEntry(self, entry, documentId):
"""If True, insert directly into treeset
"""
if entry:
self._index.insert(documentId)
self._length.change(1)
def removeForwardIndexEntry(self, entry, documentId):
"""Take the entry provided and remove any reference to documentId
in its entry in the index.
"""
try:
if entry:
self._index.remove(documentId)
self._length.change(-1)
except ConflictError:
raise
except Exception:
LOG.exception('%s: unindex_object could not remove '
'documentId %s from index %s. This '
'should not happen.' % (self.__class__.__name__,
str(documentId), str(self.id)))
def _index_object(self, documentId, obj, threshold=None, attr=''):
""" index and object 'obj' with integer id 'documentId'"""
returnStatus = 0
# First we need to see if there's anything interesting to look at
datum = self._get_object_datum(obj, attr)
# Make it boolean, int as an optimization
datum = int(bool(datum))
# We don't want to do anything that we don't have to here, so we'll
# check to see if the new and existing information is the same.
oldDatum = self._unindex.get(documentId, _marker)
if datum != oldDatum:
if oldDatum is not _marker:
self.removeForwardIndexEntry(oldDatum, documentId)
if datum is _marker:
try:
del self._unindex[documentId]
except ConflictError:
raise
except Exception:
LOG.error('Should not happen: oldDatum was there, now '
'its not, for document with id %s' %
documentId)
if datum is not _marker:
if datum:
self.insertForwardIndexEntry(datum, documentId)
self._unindex[documentId] = datum
returnStatus = 1
return returnStatus
def _apply_index(self, request, resultset=None):
record = parseIndexRequest(request, self.id, self.query_options)
if record.keys is None:
return None
index = self._index
for key in record.keys:
if key:
# If True, check index
return (intersection(index, resultset), (self.id, ))
else:
# Otherwise, remove from resultset or _unindex
if resultset is None:
return (union(difference(self._unindex, index), IISet([])),
(self.id, ))
else:
return (difference(resultset, index), (self.id, ))
return (IISet(), (self.id, ))
def indexSize(self):
"""Return distinct values, as an optimization we always claim 2."""
return 2
def items(self):
items = []
for v, k in self._unindex.items():
if isinstance(v, int):
v = IISet((v, ))
items.append((k, v))
return items
manage_addBooleanIndexForm = DTMLFile('dtml/addBooleanIndex', globals())
def manage_addBooleanIndex(self, id, extra=None,
REQUEST=None, RESPONSE=None, URL3=None):
"""Add a boolean index"""
return self.manage_addIndex(id, 'BooleanIndex', extra=extra, \
REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)
src/Products/PluginIndexes/BooleanIndex/dtml/addBooleanIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var "manage_form_title(this(), _, form_title='Add BooleanIndex')">
<p class="form-help">
<strong>Boolean Indexes</strong> can be used for keeping track of
whether objects fulfills a certain contract, like isFolderish
</p>
<form action="manage_addBooleanIndex" method="post" enctype="multipart/form-data">
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Id
</div>
</td>
<td align="left" valign="top">
<input type="text" name="id" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-label">
Indexed attributes
</div>
</td>
<td align="left" valign="top">
<input type="text" name="extra.indexed_attrs:record:string" size="40" />
<em>attribute1,attribute2,...</em> or leave empty
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-optional">
Type
</div>
</td>
<td align="left" valign="top">
Boolean Index
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/BooleanIndex/dtml/manageBooleanIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p class="form-help">
Objects indexed: <dtml-var numObjects>
<br>
Distinct values: <dtml-var indexSize>
</p>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/BooleanIndex/tests.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2010 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import unittest
from BTrees.IIBTree import IISet
class Dummy(object):
def __init__(self, docid, truth):
self.id = docid
self.truth = truth
class TestBooleanIndex(unittest.TestCase):
def _getTargetClass(self):
from Products.PluginIndexes.BooleanIndex import BooleanIndex
return BooleanIndex.BooleanIndex
def _makeOne(self, attr='truth'):
return self._getTargetClass()(attr)
def test_index_true(self):
index = self._makeOne()
obj = Dummy(1, True)
index._index_object(obj.id, obj, attr='truth')
self.failUnless(1 in index._unindex)
self.failUnless(1 in index._index)
def test_index_false(self):
index = self._makeOne()
obj = Dummy(1, False)
index._index_object(obj.id, obj, attr='truth')
self.failUnless(1 in index._unindex)
self.failIf(1 in index._index)
def test_search_true(self):
index = self._makeOne()
obj = Dummy(1, True)
index._index_object(obj.id, obj, attr='truth')
obj = Dummy(2, False)
index._index_object(obj.id, obj, attr='truth')
res, idx = index._apply_index({'truth': True})
self.failUnlessEqual(idx, ('truth', ))
self.failUnlessEqual(list(res), [1])
def test_search_false(self):
index = self._makeOne()
obj = Dummy(1, True)
index._index_object(obj.id, obj, attr='truth')
obj = Dummy(2, False)
index._index_object(obj.id, obj, attr='truth')
res, idx = index._apply_index({'truth': False})
self.failUnlessEqual(idx, ('truth', ))
self.failUnlessEqual(list(res), [2])
def test_search_inputresult(self):
index = self._makeOne()
obj = Dummy(1, True)
index._index_object(obj.id, obj, attr='truth')
obj = Dummy(2, False)
index._index_object(obj.id, obj, attr='truth')
res, idx = index._apply_index({'truth': True}, resultset=IISet([]))
self.failUnlessEqual(idx, ('truth', ))
self.failUnlessEqual(list(res), [])
res, idx = index._apply_index({'truth': True}, resultset=IISet([2]))
self.failUnlessEqual(idx, ('truth', ))
self.failUnlessEqual(list(res), [])
res, idx = index._apply_index({'truth': True}, resultset=IISet([1]))
self.failUnlessEqual(idx, ('truth', ))
self.failUnlessEqual(list(res), [1])
res, idx = index._apply_index({'truth': True}, resultset=IISet([1, 2]))
self.failUnlessEqual(idx, ('truth', ))
self.failUnlessEqual(list(res), [1])
res, idx = index._apply_index({'truth': False},
resultset=IISet([1, 2]))
self.failUnlessEqual(idx, ('truth', ))
self.failUnlessEqual(list(res), [2])
def test_suite():
from unittest import TestSuite, makeSuite
suite = TestSuite()
suite.addTest(makeSuite(TestBooleanIndex))
return suite
src/Products/PluginIndexes/DateIndex/DateIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Date index.
"""
import time
from logging import getLogger
from datetime import date, datetime
from datetime import tzinfo, timedelta
from App.special_dtml import DTMLFile
from BTrees.IIBTree import IISet
from BTrees.IIBTree import union
from BTrees.IIBTree import intersection
from BTrees.IIBTree import multiunion
from BTrees.IOBTree import IOBTree
from BTrees.Length import Length
from BTrees.OIBTree import OIBTree
from DateTime.DateTime import DateTime
from OFS.PropertyManager import PropertyManager
from ZODB.POSException import ConflictError
from zope.interface import implements
from Products.PluginIndexes.common import safe_callable
from Products.PluginIndexes.common.UnIndex import UnIndex
from Products.PluginIndexes.common.util import parseIndexRequest
from Products.PluginIndexes.interfaces import IDateIndex
LOG = getLogger('DateIndex')
_marker = []
###############################################################################
# copied from Python 2.3 datetime.tzinfo docs
# A class capturing the platform's idea of local time.
ZERO = timedelta(0)
STDOFFSET = timedelta(seconds = -time.timezone)
if time.daylight:
DSTOFFSET = timedelta(seconds = -time.altzone)
else:
DSTOFFSET = STDOFFSET
DSTDIFF = DSTOFFSET - STDOFFSET
MAX32 = int(2**31 - 1)
class LocalTimezone(tzinfo):
def utcoffset(self, dt):
if self._isdst(dt):
return DSTOFFSET
else:
return STDOFFSET
def dst(self, dt):
if self._isdst(dt):
return DSTDIFF
else:
return ZERO
def tzname(self, dt):
return time.tzname[self._isdst(dt)]
def _isdst(self, dt):
tt = (dt.year, dt.month, dt.day,
dt.hour, dt.minute, dt.second,
dt.weekday(), 0, -1)
stamp = time.mktime(tt)
tt = time.localtime(stamp)
return tt.tm_isdst > 0
Local = LocalTimezone()
###############################################################################
class DateIndex(UnIndex, PropertyManager):
"""Index for dates.
"""
implements(IDateIndex)
meta_type = 'DateIndex'
query_options = ('query', 'range')
index_naive_time_as_local = True # False means index as UTC
_properties=({'id':'index_naive_time_as_local',
'type':'boolean',
'mode':'w'},)
manage = manage_main = DTMLFile( 'dtml/manageDateIndex', globals() )
manage_browse = DTMLFile('../dtml/browseIndex', globals())
manage_main._setName( 'manage_main' )
manage_options = ( { 'label' : 'Settings'
, 'action' : 'manage_main'
},
{'label': 'Browse',
'action': 'manage_browse',
},
) + PropertyManager.manage_options
def clear( self ):
""" Complete reset """
self._index = IOBTree()
self._unindex = OIBTree()
self._length = Length()
def index_object( self, documentId, obj, threshold=None ):
"""index an object, normalizing the indexed value to an integer
o Normalized value has granularity of one minute.
o Objects which have 'None' as indexed value are *omitted*,
by design.
"""
returnStatus = 0
try:
date_attr = getattr( obj, self.id )
if safe_callable( date_attr ):
date_attr = date_attr()
ConvertedDate = self._convert( value=date_attr, default=_marker )
except AttributeError:
ConvertedDate = _marker
oldConvertedDate = self._unindex.get( documentId, _marker )
if ConvertedDate != oldConvertedDate:
if oldConvertedDate is not _marker:
self.removeForwardIndexEntry(oldConvertedDate, documentId)
if ConvertedDate is _marker:
try:
del self._unindex[documentId]
except ConflictError:
raise
except:
LOG.error("Should not happen: ConvertedDate was there,"
" now it's not, for document with id %s" %
documentId)
if ConvertedDate is not _marker:
self.insertForwardIndexEntry( ConvertedDate, documentId )
self._unindex[documentId] = ConvertedDate
returnStatus = 1
return returnStatus
def _apply_index(self, request, resultset=None):
"""Apply the index to query parameters given in the argument
Normalize the 'query' arguments into integer values at minute
precision before querying.
"""
record = parseIndexRequest(request, self.id, self.query_options)
if record.keys is None:
return None
keys = map( self._convert, record.keys )
index = self._index
r = None
opr = None
#experimental code for specifing the operator
operator = record.get( 'operator', self.useOperator )
if not operator in self.operators :
raise RuntimeError("operator not valid: %s" % operator)
# depending on the operator we use intersection or union
if operator=="or":
set_func = union
else:
set_func = intersection
# range parameter
range_arg = record.get('range',None)
if range_arg:
opr = "range"
opr_args = []
if range_arg.find("min") > -1:
opr_args.append("min")
if range_arg.find("max") > -1:
opr_args.append("max")
if record.get('usage',None):
# see if any usage params are sent to field
opr = record.usage.lower().split(':')
opr, opr_args = opr[0], opr[1:]
if opr=="range": # range search
if 'min' in opr_args:
lo = min(keys)
else:
lo = None
if 'max' in opr_args:
hi = max(keys)
else:
hi = None
if hi:
setlist = index.values(lo,hi)
else:
setlist = index.values(lo)
r = multiunion(setlist)
else: # not a range search
for key in keys:
set = index.get(key, None)
if set is not None:
if isinstance(set, int):
set = IISet((set,))
else:
# set can't be bigger than resultset
set = intersection(set, resultset)
r = set_func(r, set)
if isinstance(r, int):
r = IISet((r,))
if r is None:
return IISet(), (self.id,)
else:
return r, (self.id,)
def _convert( self, value, default=None ):
"""Convert Date/Time value to our internal representation"""
# XXX: Code patched 20/May/2003 by Kiran Jonnalagadda to
# convert dates to UTC first.
if isinstance(value, DateTime):
t_tup = value.toZone('UTC').parts()
elif isinstance(value, (float, int)):
t_tup = time.gmtime( value )
elif isinstance(value, str) and value:
t_obj = DateTime( value ).toZone('UTC')
t_tup = t_obj.parts()
elif isinstance(value, datetime):
if self.index_naive_time_as_local and value.tzinfo is None:
value = value.replace(tzinfo=Local)
# else if tzinfo is None, naive time interpreted as UTC
t_tup = value.utctimetuple()
elif isinstance(value, date):
t_tup = value.timetuple()
else:
return default
yr = t_tup[0]
mo = t_tup[1]
dy = t_tup[2]
hr = t_tup[3]
mn = t_tup[4]
t_val = ( ( ( ( yr * 12 + mo ) * 31 + dy ) * 24 + hr ) * 60 + mn )
if t_val > MAX32:
# t_val must be integer fitting in the 32bit range
raise OverflowError(
"%s is not within the range of indexable dates (index: %s)"
% (value, self.id))
return t_val
manage_addDateIndexForm = DTMLFile( 'dtml/addDateIndex', globals() )
def manage_addDateIndex( self, id, REQUEST=None, RESPONSE=None, URL3=None):
"""Add a Date index"""
return self.manage_addIndex(id, 'DateIndex', extra=None, \
REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)
src/Products/PluginIndexes/DateIndex/README.txt deleted 100644 → 0
View file @ 03a5df08
DateIndex README
Overview
Normal FieldIndexes *can* be used to index values which are DateTime
instances, but they are hideously expensive:
o DateTime instances are *huge*, both in RAM and on disk.
o DateTime instances maintain an absurd amount of precision, far
beyond any reasonable search criteria for "normal" cases.
DateIndex is a pluggable index which addresses these two issues
as follows:
o It normalizes the indexed value to an integer representation
with a granularity of one minute.
o It normalizes the 'query' value into the same form.
o Objects which return 'None' for the index query are omitted from
the index.
src/Products/PluginIndexes/DateIndex/dtml/addDateIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var "manage_form_title(this(), _,
form_title='Add DateIndex',
)">
<p class="form-help">
A <em>DateIndex</em> indexes DateTime attributes.
</p>
<form action="manage_addDateIndex" method="post" enctype="multipart/form-data">
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Id
</div>
</td>
<td align="left" valign="top">
<input type="text" name="id" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-optional">
Type
</div>
</td>
<td align="left" valign="top">
DateIndex
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/DateIndex/dtml/manageDateIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p class="form-help">
Objects indexed: <dtml-var numObjects>
<br>
Distinct values: <dtml-var indexSize>
</p>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/DateIndex/tests/__init__.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
# This file is needed to make this a package.
src/Products/PluginIndexes/DateIndex/tests/test_DateIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""DateIndex unit tests.
"""
import unittest
class Dummy:
def __init__(self, name, date):
self._name = name
self._date = date
def name(self):
return self._name
def date(self):
return self._date
def __str__(self):
return "<Dummy %s, date %s>" % (self._name, str(self._date))
###############################################################################
# excerpted from the Python module docs
###############################################################################
def _getEastern():
from datetime import date
from datetime import datetime
from datetime import timedelta
from datetime import tzinfo
ZERO = timedelta(0)
HOUR = timedelta(hours=1)
def first_sunday_on_or_after(dt):
days_to_go = 6 - dt.weekday()
if days_to_go:
dt += timedelta(days_to_go)
return dt
# In the US, DST starts at 2am (standard time) on the first Sunday in
# April...
DSTSTART = datetime(1, 4, 1, 2)
# and ends at 2am (DST time; 1am standard time) on the last Sunday of
# October, which is the first Sunday on or after Oct 25.
DSTEND = datetime(1, 10, 25, 1)
class USTimeZone(tzinfo):
def __init__(self, hours, reprname, stdname, dstname):
self.stdoffset = timedelta(hours=hours)
self.reprname = reprname
self.stdname = stdname
self.dstname = dstname
def __repr__(self):
return self.reprname
def tzname(self, dt):
if self.dst(dt):
return self.dstname
else:
return self.stdname
def utcoffset(self, dt):
return self.stdoffset + self.dst(dt)
def dst(self, dt):
if dt is None or dt.tzinfo is None:
# An exception may be sensible here, in one or both cases.
# It depends on how you want to treat them. The default
# fromutc() implementation (called by the default astimezone()
# implementation) passes a datetime with dt.tzinfo is self.
return ZERO
assert dt.tzinfo is self
# Find first Sunday in April & the last in October.
start = first_sunday_on_or_after(DSTSTART.replace(year=dt.year))
end = first_sunday_on_or_after(DSTEND.replace(year=dt.year))
# Can't compare naive to aware objects, so strip the timezone from
# dt first.
if start <= dt.replace(tzinfo=None) < end:
return HOUR
else:
return ZERO
return USTimeZone(-5, "Eastern", "EST", "EDT")
###############################################################################
class DI_Tests(unittest.TestCase):
def _getTargetClass(self):
from Products.PluginIndexes.DateIndex.DateIndex import DateIndex
return DateIndex
def _makeOne(self, id='date'):
return self._getTargetClass()(id)
def _getValues(self):
from DateTime import DateTime
from datetime import date
from datetime import datetime
return [
(0, Dummy('a', None)), # None
(1, Dummy('b', DateTime(0))), # 1055335680
(2, Dummy('c', DateTime('2002-05-08 15:16:17'))), # 1072667236
(3, Dummy('d', DateTime('2032-05-08 15:16:17'))), # 1088737636
(4, Dummy('e', DateTime('2062-05-08 15:16:17'))), # 1018883325
(5, Dummy('e', DateTime('2062-05-08 15:16:17'))), # 1018883325
(6, Dummy('f', 1072742620.0)), # 1073545923
(7, Dummy('f', 1072742900)), # 1073545928
(8, Dummy('g', date(2034,2,5))), # 1073599200
(9, Dummy('h', datetime(2034,2,5,15,20,5))), # (varies)
(10, Dummy('i', datetime(2034,2,5,10,17,5,
tzinfo=_getEastern()))), # 1073600117
]
def _populateIndex(self, index):
for k, v in self._getValues():
index.index_object(k, v)
def _checkApply(self, index, req, expectedValues):
result, used = index._apply_index(req)
if hasattr(result, 'keys'):
result = result.keys()
self.assertEqual(used, ('date',))
self.assertEqual(len(result), len(expectedValues),
'%s | %s' % (result, expectedValues))
for k, v in expectedValues:
self.assertTrue(k in result)
def _convert(self, dt):
from time import gmtime
from datetime import date
from datetime import datetime
from Products.PluginIndexes.DateIndex.DateIndex import Local
if isinstance(dt, (float, int)):
yr, mo, dy, hr, mn = gmtime(dt)[:5]
elif type(dt) is date:
yr, mo, dy, hr, mn = dt.timetuple()[:5]
elif type(dt) is datetime:
if dt.tzinfo is None: # default behavior of index
dt = dt.replace(tzinfo=Local)
yr, mo, dy, hr, mn = dt.utctimetuple()[:5]
else:
yr, mo, dy, hr, mn = dt.toZone('UTC').parts()[:5]
return (((yr * 12 + mo) * 31 + dy) * 24 + hr) * 60 + mn
def test_interfaces(self):
from Products.PluginIndexes.interfaces import IDateIndex
from Products.PluginIndexes.interfaces import IPluggableIndex
from Products.PluginIndexes.interfaces import ISortIndex
from Products.PluginIndexes.interfaces import IUniqueValueIndex
from zope.interface.verify import verifyClass
verifyClass(IDateIndex, self._getTargetClass())
verifyClass(IPluggableIndex, self._getTargetClass())
verifyClass(ISortIndex, self._getTargetClass())
verifyClass(IUniqueValueIndex, self._getTargetClass())
def test_empty(self):
from DateTime import DateTime
index = self._makeOne()
self.assertEqual(len(index), 0)
self.assertEqual(len(index.referencedObjects()), 0)
self.assertTrue(index.getEntryForObject(1234) is None)
marker = []
self.assertTrue(index.getEntryForObject(1234, marker) is marker)
index.unindex_object(1234) # shouldn't throw
self.assertTrue(index.hasUniqueValuesFor('date'))
self.assertFalse(index.hasUniqueValuesFor('foo'))
self.assertEqual(len(index.uniqueValues('date')), 0)
self.assertTrue(index._apply_index({'zed': 12345}) is None)
self._checkApply(index,
{'date': DateTime(0)}, [])
self._checkApply(index,
{'date': {'query': DateTime('2032-05-08 15:16:17'),
'range': 'min'}},
[])
self._checkApply(index,
{'date': {'query': DateTime('2032-05-08 15:16:17'),
'range': 'max'}},
[])
self._checkApply(index,
{'date': {'query':(DateTime('2002-05-08 15:16:17'),
DateTime('2062-05-08 15:16:17')),
'range': 'min:max'}},
[])
def test_retrieval( self ):
from DateTime import DateTime
index = self._makeOne()
self._populateIndex(index)
values = self._getValues()
self.assertEqual(len(index), len(values) - 2) # One dupe, one empty
self.assertEqual(len(index.referencedObjects()), len(values) - 1)
# One empty
self.assertTrue(index.getEntryForObject(1234) is None)
marker = []
self.assertTrue(index.getEntryForObject(1234, marker) is marker)
index.unindex_object(1234) # shouldn't throw
for k, v in values:
if v.date():
self.assertEqual(index.getEntryForObject(k),
self._convert(v.date()))
self.assertEqual(len(index.uniqueValues('date')), len(values) - 2)
self.assertTrue(index._apply_index({'bar': 123}) is None)
self._checkApply(index,
{'date': DateTime(0)}, values[1:2])
self._checkApply(index,
{'date': {'query': DateTime('2032-05-08 15:16:17'),
'range': 'min'}},
values[3:6] + values[8:])
self._checkApply(index,
{'date': {'query': DateTime('2032-05-08 15:16:17'),
'range': 'max'}},
values[1:4] + values[6:8])
self._checkApply(index,
{'date': {'query':(DateTime('2002-05-08 15:16:17'),
DateTime('2062-05-08 15:16:17')),
'range': 'min:max'}},
values[2:] )
self._checkApply(index,
{'date': 1072742620.0}, [values[6]])
self._checkApply(index,
{'date': 1072742900}, [values[7]])
def test_naive_convert_to_utc(self):
index = self._makeOne()
values = self._getValues()
index.index_naive_time_as_local = False
self._populateIndex(index)
for k, v in values[9:]:
# assert that the timezone is effectively UTC for item 9,
# and still correct for item 10
yr, mo, dy, hr, mn = v.date().utctimetuple()[:5]
val = (((yr * 12 + mo) * 31 + dy) * 24 + hr) * 60 + mn
self.assertEqual(index.getEntryForObject(k), val)
def test_removal(self):
""" DateIndex would hand back spurious entries when used as a
sort_index, because it previously was not removing entries
from the _unindex when indexing an object with a value of
None. The catalog consults a sort_index's
documentToKeyMap() to build the brains.
"""
values = self._getValues()
index = self._makeOne()
self._populateIndex(index)
self._checkApply(index,
{'date': 1072742900}, [values[7]])
index.index_object(7, None)
self.assertFalse(7 in index.documentToKeyMap().keys())
def test_suite():
suite = unittest.TestSuite()
suite.addTest( unittest.makeSuite( DI_Tests ) )
return suite
src/Products/PluginIndexes/DateRangeIndex/DateRangeIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Date range index.
"""
import os
from datetime import datetime
from AccessControl.class_init import InitializeClass
from AccessControl.Permissions import manage_zcatalog_indexes
from AccessControl.Permissions import view
from AccessControl.SecurityInfo import ClassSecurityInfo
from Acquisition import aq_base
from Acquisition import aq_get
from Acquisition import aq_inner
from Acquisition import aq_parent
from App.Common import package_home
from App.special_dtml import DTMLFile
from BTrees.IIBTree import IISet
from BTrees.IIBTree import IITreeSet
from BTrees.IIBTree import difference
from BTrees.IIBTree import intersection
from BTrees.IIBTree import multiunion
from BTrees.IOBTree import IOBTree
from BTrees.Length import Length
from DateTime.DateTime import DateTime
from zope.interface import implements
from Products.PluginIndexes.common import safe_callable
from Products.PluginIndexes.common.UnIndex import UnIndex
from Products.PluginIndexes.common.util import parseIndexRequest
from Products.PluginIndexes.interfaces import IDateRangeIndex
_dtmldir = os.path.join( package_home( globals() ), 'dtml' )
MAX32 = int(2**31 - 1)
class RequestCache(dict):
def __str__(self):
return "<RequestCache %s items>" % len(self)
class DateRangeIndex(UnIndex):
"""Index for date ranges, such as the "effective-expiration" range in CMF.
Any object may return None for either the start or the end date: for the
start date, this should be the logical equivalent of "since the beginning
of time"; for the end date, "until the end of time".
Therefore, divide the space of indexed objects into four containers:
- Objects which always match (i.e., they returned None for both);
- Objects which match after a given time (i.e., they returned None for the
end date);
- Objects which match until a given time (i.e., they returned None for the
start date);
- Objects which match only during a specific interval.
"""
implements(IDateRangeIndex)
security = ClassSecurityInfo()
meta_type = "DateRangeIndex"
query_options = ('query',)
manage_options= ( { 'label' : 'Properties'
, 'action' : 'manage_indexProperties'
}
,
)
since_field = until_field = None
def __init__(self, id, since_field=None, until_field=None,
caller=None, extra=None):
if extra:
since_field = extra.since_field
until_field = extra.until_field
self._setId(id)
self._edit(since_field, until_field)
self.clear()
security.declareProtected(view, 'getSinceField')
def getSinceField(self):
"""Get the name of the attribute indexed as start date.
"""
return self._since_field
security.declareProtected(view, 'getUntilField')
def getUntilField(self):
"""Get the name of the attribute indexed as end date.
"""
return self._until_field
manage_indexProperties = DTMLFile( 'manageDateRangeIndex', _dtmldir )
security.declareProtected(manage_zcatalog_indexes, 'manage_edit')
def manage_edit( self, since_field, until_field, REQUEST ):
"""
"""
self._edit( since_field, until_field )
REQUEST[ 'RESPONSE' ].redirect( '%s/manage_main'
'?manage_tabs_message=Updated'
% REQUEST.get('URL2')
)
security.declarePrivate('_edit')
def _edit( self, since_field, until_field ):
"""
Update the fields used to compute the range.
"""
self._since_field = since_field
self._until_field = until_field
security.declareProtected(manage_zcatalog_indexes, 'clear')
def clear( self ):
"""
Start over fresh.
"""
self._always = IITreeSet()
self._since_only = IOBTree()
self._until_only = IOBTree()
self._since = IOBTree()
self._until = IOBTree()
self._unindex = IOBTree() # 'datum' will be a tuple of date ints
self._length = Length()
#
# PluggableIndexInterface implementation (XXX inherit assertions?)
#
def getEntryForObject( self, documentId, default=None ):
"""
Get all information contained for the specific object
identified by 'documentId'. Return 'default' if not found.
"""
return self._unindex.get( documentId, default )
def index_object( self, documentId, obj, threshold=None ):
"""
Index an object:
- 'documentId' is the integer ID of the document
- 'obj' is the object to be indexed
- ignore threshold
"""
if self._since_field is None:
return 0
since = getattr( obj, self._since_field, None )
if safe_callable( since ):
since = since()
since = self._convertDateTime( since )
until = getattr( obj, self._until_field, None )
if safe_callable( until ):
until = until()
until = self._convertDateTime( until )
datum = ( since, until )
old_datum = self._unindex.get( documentId, None )
if datum == old_datum: # No change? bail out!
return 0
if old_datum is not None:
old_since, old_until = old_datum
self._removeForwardIndexEntry( old_since, old_until, documentId )
self._insertForwardIndexEntry( since, until, documentId )
self._unindex[ documentId ] = datum
return 1
def unindex_object( self, documentId ):
"""
Remove the object corresponding to 'documentId' from the index.
"""
datum = self._unindex.get( documentId, None )
if datum is None:
return
since, until = datum
self._removeForwardIndexEntry( since, until, documentId )
del self._unindex[ documentId ]
def uniqueValues( self, name=None, withLengths=0 ):
"""
Return a list of unique values for 'name'.
If 'withLengths' is true, return a sequence of tuples, in
the form '( value, length )'.
"""
if not name in ( self._since_field, self._until_field ):
return []
if name == self._since_field:
t1 = self._since
t2 = self._since_only
else:
t1 = self._until
t2 = self._until_only
result = []
if not withLengths:
result.extend( t1.keys() )
result.extend( t2.keys() )
else:
for key in t1.keys():
set = t1[ key ]
if isinstance(set, int):
length = 1
else:
length = len( set )
result.append( ( key, length) )
for key in t2.keys():
set = t2[ key ]
if isinstance(set, int):
length = 1
else:
length = len( set )
result.append( ( key, length) )
return tuple( result )
def _cache_key(self, catalog):
cid = catalog.getId()
counter = getattr(aq_base(catalog), 'getCounter', None)
if counter is not None:
return '%s_%s' % (cid, counter())
return cid
def _apply_index(self, request, resultset=None):
"""
Apply the index to query parameters given in 'request', which
should be a mapping object.
If the request does not contain the needed parameters, then
return None.
Otherwise return two objects. The first object is a ResultSet
containing the record numbers of the matching records. The
second object is a tuple containing the names of all data fields
used.
"""
iid = self.id
record = parseIndexRequest(request, iid, self.query_options)
if record.keys is None:
return None
term = self._convertDateTime(record.keys[0])
REQUEST = aq_get(self, 'REQUEST', None)
if REQUEST is not None:
catalog = aq_parent(aq_parent(aq_inner(self)))
if catalog is not None:
key = self._cache_key(catalog)
cache = REQUEST.get(key, None)
tid = term / 10
if resultset is None:
cachekey = '_daterangeindex_%s_%s' % (iid, tid)
else:
cachekey = '_daterangeindex_inverse_%s_%s' % (iid, tid)
if cache is None:
cache = REQUEST[key] = RequestCache()
else:
cached = cache.get(cachekey, None)
if cached is not None:
if resultset is None:
return (cached,
(self._since_field, self._until_field))
else:
return (difference(resultset, cached),
(self._since_field, self._until_field))
if resultset is None:
# Aggregate sets for each bucket separately, to avoid
# large-small union penalties.
until_only = multiunion(self._until_only.values(term))
since_only = multiunion(self._since_only.values(None, term))
until = multiunion(self._until.values(term))
# Total result is bound by resultset
if REQUEST is None:
until = intersection(resultset, until)
since = multiunion(self._since.values(None, term))
bounded = intersection(until, since)
# Merge from smallest to largest.
result = multiunion([bounded, until_only, since_only,
self._always])
if REQUEST is not None and catalog is not None:
cache[cachekey] = result
return (result, (self._since_field, self._until_field))
else:
# Compute the inverse and subtract from res
until_only = multiunion(self._until_only.values(None, term))
since_only = multiunion(self._since_only.values(term))
until = multiunion(self._until.values(None, term))
since = multiunion(self._since.values(term))
result = multiunion([until_only, since_only, until,since])
if REQUEST is not None and catalog is not None:
cache[cachekey] = result
return (difference(resultset, result),
(self._since_field, self._until_field))
def _insertForwardIndexEntry( self, since, until, documentId ):
"""
Insert 'documentId' into the appropriate set based on
'datum'.
"""
if since is None and until is None:
self._always.insert( documentId )
elif since is None:
set = self._until_only.get( until, None )
if set is None:
self._until_only[ until ] = documentId
else:
if isinstance(set, (int, IISet)):
set = self._until_only[until] = IITreeSet((set, documentId))
else:
set.insert( documentId )
elif until is None:
set = self._since_only.get( since, None )
if set is None:
self._since_only[ since ] = documentId
else:
if isinstance(set, (int, IISet)):
set = self._since_only[since] = IITreeSet((set, documentId))
else:
set.insert( documentId )
else:
set = self._since.get( since, None )
if set is None:
self._since[ since ] = documentId
else:
if isinstance(set, (int, IISet)):
set = self._since[since] = IITreeSet((set, documentId))
else:
set.insert( documentId )
set = self._until.get( until, None )
if set is None:
self._until[ until ] = documentId
else:
if isinstance(set, (int, IISet)):
set = self._until[until] = IITreeSet((set, documentId))
else:
set.insert( documentId )
def _removeForwardIndexEntry( self, since, until, documentId ):
"""
Remove 'documentId' from the appropriate set based on
'datum'.
"""
if since is None and until is None:
self._always.remove( documentId )
elif since is None:
set = self._until_only.get( until, None )
if set is not None:
if isinstance(set, int):
del self._until_only[until]
else:
set.remove( documentId )
if not set:
del self._until_only[ until ]
elif until is None:
set = self._since_only.get( since, None )
if set is not None:
if isinstance(set, int):
del self._since_only[ since ]
else:
set.remove( documentId )
if not set:
del self._since_only[ since ]
else:
set = self._since.get( since, None )
if set is not None:
if isinstance(set, int):
del self._since[ since ]
else:
set.remove( documentId )
if not set:
del self._since[ since ]
set = self._until.get( until, None )
if set is not None:
if isinstance(set, int):
del self._until[ until ]
else:
set.remove( documentId )
if not set:
del self._until[ until ]
def _convertDateTime( self, value ):
if value is None:
return value
if isinstance(value, (str, datetime)):
dt_obj = DateTime(value)
value = dt_obj.millis() / 1000 / 60 # flatten to minutes
elif isinstance(value, DateTime):
value = value.millis() / 1000 / 60 # flatten to minutes
result = int( value )
if result > MAX32:
# t_val must be integer fitting in the 32bit range
raise OverflowError( '%s is not within the range of dates allowed'
'by a DateRangeIndex' % value)
return result
InitializeClass( DateRangeIndex )
manage_addDateRangeIndexForm = DTMLFile( 'addDateRangeIndex', _dtmldir )
def manage_addDateRangeIndex(self, id, extra=None,
REQUEST=None, RESPONSE=None, URL3=None):
"""
Add a date range index to the catalog, using the incredibly icky
double-indirection-which-hides-NOTHING.
"""
return self.manage_addIndex(id, 'DateRangeIndex', extra,
REQUEST, RESPONSE, URL3)
src/Products/PluginIndexes/DateRangeIndex/README.txt deleted 100644 → 0
View file @ 03a5df08
DateRangeIndex README
Overview
Zope applications frequently wish to perform efficient queries
against a pair of date attributes/methods, representing a time
interval (e.g., effective / expiration dates). This query *can*
be done using a pair of indexes, but this implementation is
hideously expensive:
o DateTime instances are *huge*, both in RAM and on disk.
o DateTime instances maintain an absurd amount of precision, far
beyond any reasonable search criteria for "normal" cases.
o Results must be fetched and intersected between two indexes.
o Handling objects which do not specify both endpoints (i.e.,
where the interval is open or half-open) is iffy, as the
default value needs to be coerced into a different abnormal
value for each end to permit ordered comparison.
o The *very* common case of the open interval (neither endpoint
specified) should be optimized.
DateRangeIndex is a pluggable index which addresses these issues
as follows:
o It groups the "open" case into a special set, '_always'.
o It maintains separate ordered sets for each of the "half-open"
cases.
o It performs the expensive "intersect two range search" operation
only on the (usually small) set of objects which provide a
closed interval.
o It flattens the key values into integers with granularity of
one minute.
o It normalizes the 'query' value into the same form.
src/Products/PluginIndexes/DateRangeIndex/dtml/addDateRangeIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var "manage_form_title(this(), _,
form_title='Add DateRangeIndex')">
<p class="form-help">
A DateRangeIndex takes the name of two input attributes; one containing the
start date of the range, the second the end of the range. This index is filled
with range information based on those two markers. You can then search for
objects for those where a given date falls within the range.
</p>
<form action="manage_addDateRangeIndex" method="POST">
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Id
</div>
</td>
<td align="left" valign="top">
<input type="text" name="id" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-label">
Since field
</div>
</td>
<td align="left" valign="top">
<input type="text" name="extra.since_field:record" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-label">
Until field
</div>
</td>
<td align="left" valign="top">
<input type="text" name="extra.until_field:record" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/DateRangeIndex/dtml/manageDateRangeIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p class="form-help">
You can update this DateRangeIndex by editing the following field and clicking
<emUpdate</em>.
</p>
<p>
Objects indexed: <dtml-var numObjects>
<br>
Distinct values: <dtml-var indexSize>
</p>
<form action="&dtml-URL1;/manage_edit" method="POST">
<table cellpadding="2" cellspacing="0" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Since field
</td>
<td align="left" valign="top">
<input name="since_field" value="&dtml-getSinceField;" size="40" />
</td>
</tr>
<td align="left" valign="top">
<div class="form-label">
Until field
</td>
<td align="left" valign="top">
<input name="until_field" value="&dtml-getUntilField;" />
</td>
</tr>
<tr>
<td></td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value="Update">
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/DateRangeIndex/tests/__init__.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
# This file is needed to make this a package.
src/Products/PluginIndexes/DateRangeIndex/tests/test_DateRangeIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""DateRangeIndex unit tests.
"""
import unittest
class Dummy:
def __init__( self, name, start, stop ):
self._name = name
self._start = start
self._stop = stop
def name( self ):
return self._name
def start( self ):
return self._start
def stop( self ):
return self._stop
def datum( self ):
return ( self._start, self._stop )
dummies = [ Dummy( 'a', None, None )
, Dummy( 'b', None, None )
, Dummy( 'c', 0, None )
, Dummy( 'd', 10, None )
, Dummy( 'e', None, 4 )
, Dummy( 'f', None, 11 )
, Dummy( 'g', 0, 11 )
, Dummy( 'h', 2, 9 )
]
def matchingDummies( value ):
result = []
for dummy in dummies:
if ( ( dummy.start() is None or dummy.start() <= value )
and ( dummy.stop() is None or dummy.stop() >= value )
):
result.append( dummy )
return result
class DRI_Tests( unittest.TestCase ):
def _getTargetClass(self):
from Products.PluginIndexes.DateRangeIndex.DateRangeIndex \
import DateRangeIndex
return DateRangeIndex
def _makeOne(self,
id,
since_field=None,
until_field=None,
caller=None,
extra=None,
):
klass = self._getTargetClass()
return klass(id, since_field, until_field, caller, extra)
def test_interfaces(self):
from Products.PluginIndexes.interfaces import IDateRangeIndex
from Products.PluginIndexes.interfaces import IPluggableIndex
from Products.PluginIndexes.interfaces import ISortIndex
from Products.PluginIndexes.interfaces import IUniqueValueIndex
from zope.interface.verify import verifyClass
verifyClass(IDateRangeIndex, self._getTargetClass())
verifyClass(IPluggableIndex, self._getTargetClass())
verifyClass(ISortIndex, self._getTargetClass())
verifyClass(IUniqueValueIndex, self._getTargetClass())
def test_empty( self ):
empty = self._makeOne( 'empty' )
self.assertTrue(empty.getEntryForObject( 1234 ) is None)
empty.unindex_object( 1234 ) # shouldn't throw
self.assertFalse(empty.uniqueValues( 'foo' ))
self.assertFalse(empty.uniqueValues( 'foo', 1 ))
self.assertTrue(empty._apply_index( { 'zed' : 12345 } ) is None)
result, used = empty._apply_index( { 'empty' : 12345 } )
self.assertFalse(result)
self.assertEqual(used, ( None, None ))
def test_retrieval( self ):
index = self._makeOne( 'work', 'start', 'stop' )
for i in range( len( dummies ) ):
index.index_object( i, dummies[i] )
for i in range( len( dummies ) ):
self.assertEqual(index.getEntryForObject( i ), dummies[i].datum())
for value in range( -1, 15 ):
matches = matchingDummies( value )
results, used = index._apply_index( { 'work' : value } )
self.assertEqual(used, ( 'start', 'stop' ))
self.assertEqual(len( matches ), len( results ))
matches.sort( lambda x, y: cmp( x.name(), y.name() ) )
for result, match in map( None, results, matches ):
self.assertEqual(index.getEntryForObject(result), match.datum())
def test_longdates( self ):
self.assertRaises(OverflowError, self._badlong )
def _badlong(self):
import sys
index = self._makeOne ('work', 'start', 'stop' )
bad = Dummy( 'bad', long(sys.maxint) + 1, long(sys.maxint) + 1 )
index.index_object( 0, bad )
def test_datetime(self):
from datetime import datetime
from DateTime.DateTime import DateTime
from Products.PluginIndexes.DateIndex.tests.test_DateIndex \
import _getEastern
before = datetime(2009, 7, 11, 0, 0, tzinfo=_getEastern())
start = datetime(2009, 7, 13, 5, 15, tzinfo=_getEastern())
between = datetime(2009, 7, 13, 5, 45, tzinfo=_getEastern())
stop = datetime(2009, 7, 13, 6, 30, tzinfo=_getEastern())
after = datetime(2009, 7, 14, 0, 0, tzinfo=_getEastern())
dummy = Dummy('test', start, stop)
index = self._makeOne( 'work', 'start', 'stop' )
index.index_object(0, dummy)
self.assertEqual(index.getEntryForObject(0),
(DateTime(start).millis() / 60000,
DateTime(stop).millis() / 60000))
results, used = index._apply_index( { 'work' : before } )
self.assertEqual(len(results), 0)
results, used = index._apply_index( { 'work' : start } )
self.assertEqual(len(results), 1)
results, used = index._apply_index( { 'work' : between } )
self.assertEqual(len(results), 1)
results, used = index._apply_index( { 'work' : stop } )
self.assertEqual(len(results), 1)
results, used = index._apply_index( { 'work' : after } )
self.assertEqual(len(results), 0)
def test_datetime_naive_timezone(self):
from datetime import datetime
from DateTime.DateTime import DateTime
from Products.PluginIndexes.DateIndex.DateIndex import Local
before = datetime(2009, 7, 11, 0, 0)
start = datetime(2009, 7, 13, 5, 15)
start_local = datetime(2009, 7, 13, 5, 15, tzinfo=Local)
between = datetime(2009, 7, 13, 5, 45)
stop = datetime(2009, 7, 13, 6, 30)
stop_local = datetime(2009, 7, 13, 6, 30, tzinfo=Local)
after = datetime(2009, 7, 14, 0, 0)
dummy = Dummy('test', start, stop)
index = self._makeOne( 'work', 'start', 'stop' )
index.index_object(0, dummy)
self.assertEqual(index.getEntryForObject(0),
(DateTime(start_local).millis() / 60000,
DateTime(stop_local).millis() / 60000))
results, used = index._apply_index( { 'work' : before } )
self.assertEqual(len(results), 0)
results, used = index._apply_index( { 'work' : start } )
self.assertEqual(len(results), 1)
results, used = index._apply_index( { 'work' : between } )
self.assertEqual(len(results), 1)
results, used = index._apply_index( { 'work' : stop } )
self.assertEqual(len(results), 1)
results, used = index._apply_index( { 'work' : after } )
self.assertEqual(len(results), 0)
def test_suite():
suite = unittest.TestSuite()
suite.addTest( unittest.makeSuite( DRI_Tests ) )
return suite
src/Products/PluginIndexes/FieldIndex/FieldIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Simple column indices.
"""
from App.special_dtml import DTMLFile
from Products.PluginIndexes.common.UnIndex import UnIndex
class FieldIndex(UnIndex):
"""Index for simple fields.
"""
meta_type="FieldIndex"
manage_options= (
{'label': 'Settings', 'action': 'manage_main'},
{'label': 'Browse', 'action': 'manage_browse'},
)
query_options = ["query","range"]
manage = manage_main = DTMLFile('dtml/manageFieldIndex', globals())
manage_main._setName('manage_main')
manage_browse = DTMLFile('../dtml/browseIndex', globals())
manage_addFieldIndexForm = DTMLFile('dtml/addFieldIndex', globals())
def manage_addFieldIndex(self, id, extra=None,
REQUEST=None, RESPONSE=None, URL3=None):
"""Add a field index"""
return self.manage_addIndex(id, 'FieldIndex', extra=extra, \
REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)
src/Products/PluginIndexes/FieldIndex/dtml/addFieldIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var "manage_form_title(this(), _,
form_title='Add FieldIndex',
)">
<p class="form-help">
<strong>Field Indexes</strong> treat the value of an objects attributes
atomically, and can be used, for example, to track only a certain subset
of object values, such as 'meta_type'.
</p>
<form action="manage_addFieldIndex" method="post" enctype="multipart/form-data">
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Id
</div>
</td>
<td align="left" valign="top">
<input type="text" name="id" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-label">
Indexed attributes
</div>
</td>
<td align="left" valign="top">
<input type="text" name="extra.indexed_attrs:record:string" size="40" />
<em>attribute1,attribute2,...</em> or leave empty
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-optional">
Type
</div>
</td>
<td align="left" valign="top">
Field Index
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/FieldIndex/dtml/manageFieldIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p class="form-help">
Objects indexed: <dtml-var numObjects>
<br>
Distinct values: <dtml-var indexSize>
</p>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/FieldIndex/tests/__init__.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
# This file is needed to make this a package.
src/Products/PluginIndexes/FieldIndex/tests/testFieldIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""FieldIndex unit tests.
"""
import unittest
import Zope2
Zope2.startup()
from Products.PluginIndexes.FieldIndex.FieldIndex import FieldIndex
class Dummy:
def __init__( self, foo ):
self._foo = foo
def foo( self ):
return self._foo
def __str__( self ):
return '<Dummy: %s>' % self._foo
__repr__ = __str__
class FieldIndexTests(unittest.TestCase):
"""Test FieldIndex objects.
"""
def setUp( self ):
"""
"""
self._index = FieldIndex( 'foo' )
self._marker = []
self._values = [ ( 0, Dummy( 'a' ) )
, ( 1, Dummy( 'ab' ) )
, ( 2, Dummy( 'abc' ) )
, ( 3, Dummy( 'abca' ) )
, ( 4, Dummy( 'abcd' ) )
, ( 5, Dummy( 'abce' ) )
, ( 6, Dummy( 'abce' ) )
, ( 7, Dummy( 0 ) ) # Collector #1959
, ( 8, Dummy(None) )]
self._forward = {}
self._backward = {}
for k, v in self._values:
self._backward[k] = v
keys = self._forward.get( v, [] )
self._forward[v] = keys
self._noop_req = { 'bar': 123 }
self._request = { 'foo': 'abce' }
self._min_req = { 'foo': {'query': 'abc'
, 'range': 'min'}
}
self._max_req = { 'foo': {'query': 'abc'
, 'range': 'max' }
}
self._range_req = { 'foo': {'query': ( 'abc', 'abcd' )
, 'range': 'min:max' }
}
self._zero_req = { 'foo': 0 }
self._none_req = { 'foo': None }
def tearDown( self ):
"""
"""
def _populateIndex( self ):
for k, v in self._values:
self._index.index_object( k, v )
def _checkApply( self, req, expectedValues ):
result, used = self._index._apply_index( req )
if hasattr(result, 'keys'):
result = result.keys()
assert used == ( 'foo', )
assert len( result ) == len( expectedValues ), \
'%s | %s' % ( map( None, result ), expectedValues )
for k, v in expectedValues:
assert k in result
def test_interfaces(self):
from Products.PluginIndexes.interfaces import IPluggableIndex
from Products.PluginIndexes.interfaces import ISortIndex
from Products.PluginIndexes.interfaces import IUniqueValueIndex
from zope.interface.verify import verifyClass
verifyClass(IPluggableIndex, FieldIndex)
verifyClass(ISortIndex, FieldIndex)
verifyClass(IUniqueValueIndex, FieldIndex)
def testEmpty( self ):
"Test an empty FieldIndex."
assert len( self._index ) == 0
assert len( self._index.referencedObjects() ) == 0
self.assertEqual(self._index.numObjects(), 0)
assert self._index.getEntryForObject( 1234 ) is None
assert ( self._index.getEntryForObject( 1234, self._marker )
is self._marker )
self._index.unindex_object( 1234 ) # nothrow
assert self._index.hasUniqueValuesFor( 'foo' )
assert not self._index.hasUniqueValuesFor( 'bar' )
assert len( self._index.uniqueValues( 'foo' ) ) == 0
assert self._index._apply_index( self._noop_req ) is None
self._checkApply( self._request, [] )
self._checkApply( self._min_req, [] )
self._checkApply( self._max_req, [] )
self._checkApply( self._range_req, [] )
def testPopulated( self ):
""" Test a populated FieldIndex """
self._populateIndex()
values = self._values
assert len( self._index ) == len( values )-1 #'abce' is duplicate
assert len( self._index.referencedObjects() ) == len( values )
self.assertEqual(self._index.indexSize(), len( values )-1)
assert self._index.getEntryForObject( 1234 ) is None
assert ( self._index.getEntryForObject( 1234, self._marker )
is self._marker )
self._index.unindex_object( 1234 ) # nothrow
for k, v in values:
assert self._index.getEntryForObject( k ) == v.foo()
assert len( self._index.uniqueValues( 'foo' ) ) == len( values )-1
assert self._index._apply_index( self._noop_req ) is None
self._checkApply( self._request, values[ -4:-2 ] )
self._checkApply( self._min_req, values[ 2:-2 ] )
self._checkApply( self._max_req, values[ :3 ] + values[ -2: ] )
self._checkApply( self._range_req, values[ 2:5 ] )
def testZero( self ):
""" Make sure 0 gets indexed """
self._populateIndex()
values = self._values
self._checkApply( self._zero_req, values[ -2:-1 ] )
assert 0 in self._index.uniqueValues( 'foo' )
def testNone(self):
""" make sure None gets indexed """
self._populateIndex()
values = self._values
self._checkApply(self._none_req, values[-1:])
assert None in self._index.uniqueValues('foo')
def testReindex( self ):
self._populateIndex()
result, used = self._index._apply_index( {'foo':'abc'} )
assert list(result)==[2]
assert self._index.keyForDocument(2)=='abc'
d = Dummy('world')
self._index.index_object(2,d)
result, used = self._index._apply_index( {'foo':'world'} )
assert list(result)==[2]
assert self._index.keyForDocument(2)=='world'
del d._foo
self._index.index_object(2,d)
result, used = self._index._apply_index( {'foo':'world'} )
assert list(result)==[]
try:
should_not_be = self._index.keyForDocument(2)
except KeyError:
# As expected, we deleted that attribute.
pass
else:
# before Collector #291 this would be 'world'
raise ValueError(repr(should_not_be))
def testRange(self):
"""Test a range search"""
index = FieldIndex( 'foo' )
for i in range(100):
index.index_object(i, Dummy(i%10))
record = { 'foo' : { 'query' : [-99, 3]
, 'range' : 'min:max'
}
}
r=index._apply_index( record )
assert tuple(r[1])==('foo',), r[1]
r=list(r[0].keys())
expect=[
0, 1, 2, 3, 10, 11, 12, 13, 20, 21, 22, 23, 30, 31, 32, 33,
40, 41, 42, 43, 50, 51, 52, 53, 60, 61, 62, 63, 70, 71, 72, 73,
80, 81, 82, 83, 90, 91, 92, 93
]
assert r==expect, r
#
# Make sure that range tests with incompatible paramters
# don't return empty sets.
#
record[ 'foo' ][ 'operator' ] = 'and'
r2, ignore = index._apply_index( record )
r2 = list( r2.keys() )
assert r2 == r
def test_suite():
return unittest.TestSuite((
unittest.makeSuite(FieldIndexTests),
))
src/Products/PluginIndexes/KeywordIndex/KeywordIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Keyword index.
"""
from logging import getLogger
from BTrees.OOBTree import difference
from BTrees.OOBTree import OOSet
from App.special_dtml import DTMLFile
from Products.PluginIndexes.common import safe_callable
from Products.PluginIndexes.common.UnIndex import UnIndex
LOG = getLogger('Zope.KeywordIndex')
class KeywordIndex(UnIndex):
"""Like an UnIndex only it indexes sequences of items.
Searches match any keyword.
This should have an _apply_index that returns a relevance score
"""
meta_type="KeywordIndex"
manage_options= (
{'label': 'Settings', 'action': 'manage_main'},
{'label': 'Browse', 'action': 'manage_browse'},
)
query_options = ("query","operator", "range")
def _index_object(self, documentId, obj, threshold=None, attr=''):
""" index an object 'obj' with integer id 'i'
Ideally, we've been passed a sequence of some sort that we
can iterate over. If however, we haven't, we should do something
useful with the results. In the case of a string, this means
indexing the entire string as a keyword."""
# First we need to see if there's anything interesting to look at
# self.id is the name of the index, which is also the name of the
# attribute we're interested in. If the attribute is callable,
# we'll do so.
newKeywords = self._get_object_keywords(obj, attr)
oldKeywords = self._unindex.get(documentId, None)
if oldKeywords is None:
# we've got a new document, let's not futz around.
try:
for kw in newKeywords:
self.insertForwardIndexEntry(kw, documentId)
if newKeywords:
self._unindex[documentId] = list(newKeywords)
except TypeError:
return 0
else:
# we have an existing entry for this document, and we need
# to figure out if any of the keywords have actually changed
if type(oldKeywords) is not OOSet:
oldKeywords = OOSet(oldKeywords)
newKeywords = OOSet(newKeywords)
fdiff = difference(oldKeywords, newKeywords)
rdiff = difference(newKeywords, oldKeywords)
if fdiff or rdiff:
# if we've got forward or reverse changes
if newKeywords:
self._unindex[documentId] = list(newKeywords)
else:
del self._unindex[documentId]
if fdiff:
self.unindex_objectKeywords(documentId, fdiff)
if rdiff:
for kw in rdiff:
self.insertForwardIndexEntry(kw, documentId)
return 1
def _get_object_keywords(self, obj, attr):
newKeywords = getattr(obj, attr, ())
if safe_callable(newKeywords):
try:
newKeywords = newKeywords()
except AttributeError:
return ()
if not newKeywords:
return ()
elif isinstance(newKeywords, basestring): #Python 2.1 compat isinstance
return (newKeywords,)
else:
unique = {}
try:
for k in newKeywords:
unique[k] = None
except TypeError:
# Not a sequence
return (newKeywords,)
else:
return unique.keys()
def unindex_objectKeywords(self, documentId, keywords):
""" carefully unindex the object with integer id 'documentId'"""
if keywords is not None:
for kw in keywords:
self.removeForwardIndexEntry(kw, documentId)
def unindex_object(self, documentId):
""" carefully unindex the object with integer id 'documentId'"""
keywords = self._unindex.get(documentId, None)
self.unindex_objectKeywords(documentId, keywords)
try:
del self._unindex[documentId]
except KeyError:
LOG.debug('Attempt to unindex nonexistent'
' document id %s' % documentId)
manage = manage_main = DTMLFile('dtml/manageKeywordIndex', globals())
manage_main._setName('manage_main')
manage_browse = DTMLFile('../dtml/browseIndex', globals())
manage_addKeywordIndexForm = DTMLFile('dtml/addKeywordIndex', globals())
def manage_addKeywordIndex(self, id, extra=None,
REQUEST=None, RESPONSE=None, URL3=None):
"""Add a keyword index"""
return self.manage_addIndex(id, 'KeywordIndex', extra=extra, \
REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)
src/Products/PluginIndexes/KeywordIndex/dtml/addKeywordIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var "manage_form_title(this(), _,
form_title='Add KeywordIndex',
)">
<p class="form-help">
<strong>Keyword Indexes</strong> index a sequence of objects that act as
'keywords' for an object. A Keyword Index will return any objects
that have one or more keywords specified in a search query.
</p>
<form action="manage_addKeywordIndex" method="post" enctype="multipart/form-data">
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Id
</div>
</td>
<td align="left" valign="top">
<input type="text" name="id" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-label">
Indexed attributes
</div>
</td>
<td align="left" valign="top">
<input type="text" name="extra.indexed_attrs:record:string" size="40" />
<em>attribute1,attribute2,...</em> or leave empty
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-optional">
Type
</div>
</td>
<td align="left" valign="top">
Keyword Index
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/KeywordIndex/dtml/manageKeywordIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p class="form-help">
Objects indexed: <dtml-var numObjects>
<br>
Distinct values: <dtml-var indexSize>
</p>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/KeywordIndex/tests/__init__.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
# This file is needed to make this a package.
src/Products/PluginIndexes/KeywordIndex/tests/testKeywordIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""KeywordIndex unit tests.
"""
import unittest
import Zope2
Zope2.startup()
from Products.PluginIndexes.KeywordIndex.KeywordIndex import KeywordIndex
class Dummy:
def __init__( self, foo ):
self._foo = foo
def foo( self ):
return self._foo
def __str__( self ):
return '<Dummy: %s>' % self._foo
__repr__ = __str__
def sortedUnique(seq):
unique = {}
for i in seq:
unique[i] = None
unique = unique.keys()
unique.sort()
return unique
class TestKeywordIndex( unittest.TestCase ):
"""
Test KeywordIndex objects.
"""
_old_log_write = None
def setUp( self ):
"""
"""
self._index = KeywordIndex( 'foo' )
self._marker = []
self._values = [ ( 0, Dummy( ['a'] ) )
, ( 1, Dummy( ['a','b'] ) )
, ( 2, Dummy( ['a','b','c'] ) )
, ( 3, Dummy( ['a','b','c','a'] ) )
, ( 4, Dummy( ['a', 'b', 'c', 'd'] ) )
, ( 5, Dummy( ['a', 'b', 'c', 'e'] ) )
, ( 6, Dummy( ['a', 'b', 'c', 'e', 'f'] ))
, ( 7, Dummy( [0] ) )
]
self._noop_req = { 'bar': 123 }
self._all_req = { 'foo': ['a'] }
self._some_req = { 'foo': ['e'] }
self._overlap_req = { 'foo': ['c', 'e'] }
self._string_req = {'foo': 'a'}
self._zero_req = { 'foo': [0] }
def tearDown( self ):
"""
"""
def _populateIndex( self ):
for k, v in self._values:
self._index.index_object( k, v )
def _checkApply( self, req, expectedValues ):
result, used = self._index._apply_index( req )
assert used == ( 'foo', )
assert len(result) == len( expectedValues ), \
'%s | %s' % ( map( None, result ),
map(lambda x: x[0], expectedValues ))
if hasattr(result, 'keys'): result=result.keys()
for k, v in expectedValues:
assert k in result
def test_interfaces(self):
from Products.PluginIndexes.interfaces import IPluggableIndex
from Products.PluginIndexes.interfaces import ISortIndex
from Products.PluginIndexes.interfaces import IUniqueValueIndex
from zope.interface.verify import verifyClass
verifyClass(IPluggableIndex, KeywordIndex)
verifyClass(ISortIndex, KeywordIndex)
verifyClass(IUniqueValueIndex, KeywordIndex)
def testAddObjectWOKeywords(self):
try:
self._populateIndex()
self._index.index_object(999, None)
finally:
pass
def testEmpty( self ):
assert len( self._index ) == 0
assert len( self._index.referencedObjects() ) == 0
self.assertEqual(self._index.numObjects(), 0)
assert self._index.getEntryForObject( 1234 ) is None
assert ( self._index.getEntryForObject( 1234, self._marker )
is self._marker ), self._index.getEntryForObject(1234)
self._index.unindex_object( 1234 ) # nothrow
assert self._index.hasUniqueValuesFor( 'foo' )
assert not self._index.hasUniqueValuesFor( 'bar' )
assert len( self._index.uniqueValues( 'foo' ) ) == 0
assert self._index._apply_index( self._noop_req ) is None
self._checkApply( self._all_req, [] )
self._checkApply( self._some_req, [] )
self._checkApply( self._overlap_req, [] )
self._checkApply( self._string_req, [] )
def testPopulated( self ):
self._populateIndex()
values = self._values
#assert len( self._index ) == len( values )
assert len( self._index.referencedObjects() ) == len( values )
assert self._index.getEntryForObject( 1234 ) is None
assert ( self._index.getEntryForObject( 1234, self._marker )
is self._marker )
self._index.unindex_object( 1234 ) # nothrow
self.assertEqual(self._index.indexSize(), len( values )-1)
for k, v in values:
entry = self._index.getEntryForObject( k )
entry.sort()
kw = sortedUnique(v.foo())
self.assertEqual(entry, kw)
assert len( self._index.uniqueValues( 'foo' ) ) == len( values )-1
assert self._index._apply_index( self._noop_req ) is None
self._checkApply( self._all_req, values[:-1])
self._checkApply( self._some_req, values[ 5:7 ] )
self._checkApply( self._overlap_req, values[2:7] )
self._checkApply( self._string_req, values[:-1] )
def testZero( self ):
self._populateIndex()
values = self._values
self._checkApply( self._zero_req, values[ -1: ] )
assert 0 in self._index.uniqueValues( 'foo' )
def testReindexChange(self):
self._populateIndex()
expected = Dummy(['x', 'y'])
self._index.index_object(6, expected)
result, used = self._index._apply_index({'foo': ['x', 'y']})
result=result.keys()
assert len(result) == 1
assert result[0] == 6
result, used = self._index._apply_index(
{'foo': ['a', 'b', 'c', 'e', 'f']}
)
result = result.keys()
assert 6 not in result
def testReindexNoChange(self):
self._populateIndex()
expected = Dummy(['foo', 'bar'])
self._index.index_object(8, expected)
result, used = self._index._apply_index(
{'foo': ['foo', 'bar']})
result = result.keys()
assert len(result) == 1
assert result[0] == 8
self._index.index_object(8, expected)
result, used = self._index._apply_index(
{'foo': ['foo', 'bar']})
result = result.keys()
assert len(result) == 1
assert result[0] == 8
def testIntersectionWithRange(self):
# Test an 'and' search, ensuring that 'range' doesn't mess it up.
self._populateIndex()
record = { 'foo' : { 'query' : [ 'e', 'f' ]
, 'operator' : 'and'
}
}
self._checkApply( record, self._values[6:7] )
#
# Make sure that 'and' tests with incompatible paramters
# don't return empty sets.
#
record[ 'foo' ][ 'range' ] = 'min:max'
self._checkApply( record, self._values[6:7] )
def testDuplicateKeywords(self):
try:
self._index.index_object(0, Dummy(['a', 'a', 'b', 'b']))
self._index.unindex_object(0)
finally:
pass
def testCollectorIssue889(self) :
# Test that collector issue 889 is solved
values = self._values
nonexistent = 'foo-bar-baz'
self._populateIndex()
# make sure key is not indexed
result = self._index._index.get(nonexistent, self._marker)
assert result is self._marker
# patched _apply_index now works as expected
record = {'foo' : { 'query' : [nonexistent]
, 'operator' : 'and'}
}
self._checkApply(record, [])
record = {'foo' : { 'query' : [nonexistent, 'a']
, 'operator' : 'and'}
}
# and does not break anything
self._checkApply(record, [])
record = {'foo' : { 'query' : ['d']
, 'operator' : 'and'}
}
self._checkApply(record, values[4:5])
record = {'foo' : { 'query' : ['a', 'e']
, 'operator' : 'and'}
}
self._checkApply(record, values[5:7])
def test_noindexing_when_noattribute(self):
to_index = Dummy(['hello'])
self._index._index_object(10, to_index, attr='UNKNOWN')
self.assertFalse(self._index._unindex.get(10))
self.assertFalse(self._index.getEntryForObject(10))
def test_noindexing_when_raising_attribute(self):
class FauxObject:
def foo(self):
raise AttributeError
to_index = FauxObject()
self._index._index_object(10, to_index, attr='foo')
self.assertFalse(self._index._unindex.get(10))
self.assertFalse(self._index.getEntryForObject(10))
def test_value_removes(self):
to_index = Dummy(['hello'])
self._index._index_object(10, to_index, attr='foo')
self.assertTrue(self._index._unindex.get(10))
to_index = Dummy('')
self._index._index_object(10, to_index, attr='foo')
self.assertFalse(self._index._unindex.get(10))
def test_suite():
suite = unittest.TestSuite()
suite.addTest( unittest.makeSuite( TestKeywordIndex ) )
return suite
src/Products/PluginIndexes/PathIndex/PathIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Path index.
"""
from logging import getLogger
from App.special_dtml import DTMLFile
from OFS.SimpleItem import SimpleItem
from BTrees.IIBTree import IITreeSet
from BTrees.IIBTree import IISet
from BTrees.IIBTree import intersection
from BTrees.IIBTree import multiunion
from BTrees.IIBTree import union
from BTrees.IOBTree import IOBTree
from BTrees.OOBTree import OOBTree
from BTrees.Length import Length
from Persistence import Persistent
from zope.interface import implements
from Products.PluginIndexes.common import safe_callable
from Products.PluginIndexes.common.util import parseIndexRequest
from Products.PluginIndexes.interfaces import IPathIndex
from Products.PluginIndexes.interfaces import ISortIndex
from Products.PluginIndexes.interfaces import IUniqueValueIndex
LOG = getLogger('Zope.PathIndex')
class PathIndex(Persistent, SimpleItem):
"""Index for paths returned by getPhysicalPath.
A path index stores all path components of the physical path of an object.
Internal datastructure:
- a physical path of an object is split into its components
- every component is kept as a key of a OOBTree in self._indexes
- the value is a mapping 'level of the path component' to
'all docids with this path component on this level'
"""
implements(IPathIndex, IUniqueValueIndex, ISortIndex)
meta_type="PathIndex"
query_options = ('query', 'level', 'operator')
manage_options= (
{'label': 'Settings', 'action': 'manage_main'},
)
def __init__(self,id,caller=None):
self.id = id
self.operators = ('or','and')
self.useOperator = 'or'
self.clear()
def __len__(self):
return self._length()
# IPluggableIndex implementation
def getEntryForObject(self, docid, default=None):
""" See IPluggableIndex.
"""
try:
return self._unindex[docid]
except KeyError:
return default
def getIndexSourceNames(self):
""" See IPluggableIndex.
"""
return (self.id, 'getPhysicalPath', )
def index_object(self, docid, obj ,threshold=100):
""" See IPluggableIndex.
"""
f = getattr(obj, self.id, None)
if f is not None:
if safe_callable(f):
try:
path = f()
except AttributeError:
return 0
else:
path = f
if not isinstance(path, (str, tuple)):
raise TypeError('path value must be string or tuple of strings')
else:
try:
path = obj.getPhysicalPath()
except AttributeError:
return 0
if isinstance(path, (list, tuple)):
path = '/'+ '/'.join(path[1:])
comps = filter(None, path.split('/'))
if not self._unindex.has_key(docid):
self._length.change(1)
for i in range(len(comps)):
self.insertEntry(comps[i], docid, i)
self._unindex[docid] = path
return 1
def unindex_object(self, docid):
""" See IPluggableIndex.
"""
if docid not in self._unindex:
LOG.debug('Attempt to unindex nonexistent document with id %s'
% docid)
return
comps = self._unindex[docid].split('/')
for level in range(len(comps[1:])):
comp = comps[level+1]
try:
self._index[comp][level].remove(docid)
if not self._index[comp][level]:
del self._index[comp][level]
if not self._index[comp]:
del self._index[comp]
except KeyError:
LOG.debug('Attempt to unindex document with id %s failed'
% docid)
self._length.change(-1)
del self._unindex[docid]
def _apply_index(self, request):
""" See IPluggableIndex.
o Unpacks args from catalog and mapps onto '_search'.
"""
record = parseIndexRequest(request, self.id, self.query_options)
if record.keys is None:
return None
level = record.get("level", 0)
operator = record.get('operator', self.useOperator).lower()
# depending on the operator we use intersection of union
if operator == "or":
set_func = union
else:
set_func = intersection
res = None
for k in record.keys:
rows = self._search(k,level)
res = set_func(res,rows)
if res:
return res, (self.id,)
else:
return IISet(), (self.id,)
def numObjects(self):
""" See IPluggableIndex.
"""
return len(self._unindex)
def indexSize(self):
""" See IPluggableIndex.
"""
return len(self)
def clear(self):
""" See IPluggableIndex.
"""
self._depth = 0
self._index = OOBTree()
self._unindex = IOBTree()
self._length = Length(0)
# IUniqueValueIndex implementation
def hasUniqueValuesFor(self, name):
""" See IUniqueValueIndex.
"""
return name == self.id
def uniqueValues(self, name=None, withLength=0):
""" See IUniqueValueIndex.
"""
if name in (None, self.id, 'getPhysicalPath'):
if withLength:
for key in self._index:
yield key, len(self._search(key, -1))
else:
for key in self._index.keys():
yield key
# ISortIndex implementation
def keyForDocument(self, documentId):
""" See ISortIndex.
"""
return self._unindex.get(documentId)
def documentToKeyMap(self):
""" See ISortIndex.
"""
return self._unindex
# IPathIndex implementation.
def insertEntry(self, comp, id, level):
""" See IPathIndex
"""
if not self._index.has_key(comp):
self._index[comp] = IOBTree()
if not self._index[comp].has_key(level):
self._index[comp][level] = IITreeSet()
self._index[comp][level].insert(id)
if level > self._depth:
self._depth = level
# Helper methods
def _search(self, path, default_level=0):
""" Perform the actual search.
``path``
a string representing a relative URL, or a part of a relative URL,
or a tuple ``(path, level)``. In the first two cases, use
``default_level`` as the level for the search.
``default_level``
the level to use for non-tuple queries.
``level >= 0`` => match ``path`` only at the given level.
``level < 0`` => match ``path`` at *any* level
"""
if isinstance(path, str):
level = default_level
else:
level = int(path[1])
path = path[0]
if level < 0:
# Search at every level, return the union of all results
return multiunion(
[self._search(path, level)
for level in xrange(self._depth + 1)])
comps = filter(None, path.split('/'))
if level + len(comps) - 1 > self._depth:
# Our search is for a path longer than anything in the index
return IISet()
if len(comps) == 0:
return IISet(self._unindex.keys())
results = None
for i, comp in reversed(list(enumerate(comps))):
if not self._index.get(comp, {}).has_key(level+i): return IISet()
results = intersection(results, self._index[comp][level+i])
return results
manage = manage_main = DTMLFile('dtml/managePathIndex', globals())
manage_main._setName('manage_main')
manage_addPathIndexForm = DTMLFile('dtml/addPathIndex', globals())
def manage_addPathIndex(self, id, REQUEST=None, RESPONSE=None, URL3=None):
"""Add a path index"""
return self.manage_addIndex(id, 'PathIndex', extra=None, \
REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)
src/Products/PluginIndexes/PathIndex/PathIndex.txt deleted 100644 → 0
View file @ 03a5df08
he purpose of a PathIndex is to index Zope objects
based on their physical path. This is very similiar
to a substring search on strings.
How it works
Assume we have to index an object with id=xxx and
the physical path '/zoo/animals/africa/tiger.doc'.
We split the path into its components and keep track
of the level of every component. Inside the index we
store pairs(component,level) and the ids of the
documents::
(component,level) id of document
-----------------------------------------
('zoo',0) xxx
('animals',1) xxx
('africa',2) xxx
Note that we do not store the id of the objects itself
inside the path index.
Searching with the PathIndex
The PathIndex allows you to search for all object ids
whose objects match a physical path query. The query
is split into components and matched against the index.
E.g. '/zoo/animals' will match in the example above
but not '/zoo1/animals'. The default behaviour is to
start matching at level 0. To start matching on another
level on can specify an additional level parameter
(see API)
API
'query' -- A single or list of Path component(s) to
be searched.
'level' -- level to start searching (optional,default: 0).
If level=-1 we search through all levels.
'operator' -- either 'or' or 'and' (optional, default: 'or')
Example
Objects with the following ids and physical path should
be stored in the ZCatalog 'MyCAT'::
id physical path
----------------------------
1 /aa/bb/aa/1.txt
2 /aa/bb/bb/2.txt
3 /aa/bb/cc/3.txt
4 /bb/bb/aa/4.txt
5 /bb/bb/bb/5.txt
6 /bb/bb/cc/6.txt
7 /cc/bb/aa/7.txt
8 /cc/bb/bb/8.txt
9 /cc/bb/cc/9.txt
Query found ids
-------------------------------------------
query='/aa/bb',level=0 [1,2,3]
query='/bb/bb',level=0 [4,5,6]
query='/bb/bb',level=1 [2,5,8]
query='/bb/bb',level=-1 [2,4,5,6,8]
query='/xx' ,level=-1 []
src/Products/PluginIndexes/PathIndex/dtml/addPathIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var "manage_form_title(this(), _,
form_title='Add PathIndex',
)">
<p class="form-help">
A <em>PathIndex</em> indexes the physical path of all objects inside
a catalog. It allows you to search for objects beginning or containing
a special path component or a set of path component. A path component
is defined as <em>/&lt;component1&gt;/&lt;component2&gt;/..../&lt;object_id&gt;
</em>.
</p>
<form action="manage_addPathIndex" method="post" enctype="multipart/form-data">
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Id
</div>
</td>
<td align="left" valign="top">
<input type="text" name="id" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-optional">
Type
</div>
</td>
<td align="left" valign="top">
PathIndex
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/PathIndex/dtml/managePathIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p class="form-help">
Objects indexed: <dtml-var numObjects>
<br>
Distinct values: <dtml-var indexSize>
</p>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/PathIndex/tests/__init__.py deleted 100644 → 0
View file @ 03a5df08
# This file is needed to make this directory a package.
src/Products/PluginIndexes/PathIndex/tests/testPathIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""PathIndex unit tests.
"""
import unittest
class Dummy:
def __init__( self, path):
self.path = path
def getPhysicalPath(self):
return self.path.split('/')
DUMMIES = {
1 : Dummy("/aa/aa/aa/1.html"),
2 : Dummy("/aa/aa/bb/2.html"),
3 : Dummy("/aa/aa/cc/3.html"),
4 : Dummy("/aa/bb/aa/4.html"),
5 : Dummy("/aa/bb/bb/5.html"),
6 : Dummy("/aa/bb/cc/6.html"),
7 : Dummy("/aa/cc/aa/7.html"),
8 : Dummy("/aa/cc/bb/8.html"),
9 : Dummy("/aa/cc/cc/9.html"),
10 : Dummy("/bb/aa/aa/10.html"),
11 : Dummy("/bb/aa/bb/11.html"),
12 : Dummy("/bb/aa/cc/12.html"),
13 : Dummy("/bb/bb/aa/13.html"),
14 : Dummy("/bb/bb/bb/14.html"),
15 : Dummy("/bb/bb/cc/15.html"),
16 : Dummy("/bb/cc/aa/16.html"),
17 : Dummy("/bb/cc/bb/17.html"),
18 : Dummy("/bb/cc/cc/18.html")
}
def _populateIndex(index):
for k, v in DUMMIES.items():
index.index_object(k, v)
_marker = object()
class PathIndexTests(unittest.TestCase):
""" Test PathIndex objects """
def _getTargetClass(self):
from Products.PluginIndexes.PathIndex.PathIndex import PathIndex
return PathIndex
def _makeOne(self, id='path', caller=_marker):
if caller is not _marker:
return self._getTargetClass()(id, caller)
return self._getTargetClass()(id)
def test_class_conforms_to_IPluggableIndex(self):
from Products.PluginIndexes.interfaces import IPluggableIndex
from zope.interface.verify import verifyClass
verifyClass(IPluggableIndex, self._getTargetClass())
def test_instance_conforms_to_IPluggableIndex(self):
from Products.PluginIndexes.interfaces import IPluggableIndex
from zope.interface.verify import verifyObject
verifyObject(IPluggableIndex, self._makeOne())
def test_class_conforms_to_IUniqueValueIndex(self):
from Products.PluginIndexes.interfaces import IUniqueValueIndex
from zope.interface.verify import verifyClass
verifyClass(IUniqueValueIndex, self._getTargetClass())
def test_instance_conforms_to_IUniqueValueIndex(self):
from Products.PluginIndexes.interfaces import IUniqueValueIndex
from zope.interface.verify import verifyObject
verifyObject(IUniqueValueIndex, self._makeOne())
def test_class_conforms_to_ISortIndex(self):
from Products.PluginIndexes.interfaces import ISortIndex
from zope.interface.verify import verifyClass
verifyClass(ISortIndex, self._getTargetClass())
def test_instance_conforms_to_ISortIndex(self):
from Products.PluginIndexes.interfaces import ISortIndex
from zope.interface.verify import verifyObject
verifyObject(ISortIndex, self._makeOne())
def test_class_conforms_to_IPathIndex(self):
from Products.PluginIndexes.interfaces import IPathIndex
from zope.interface.verify import verifyClass
verifyClass(IPathIndex, self._getTargetClass())
def test_instance_conforms_to_IPathIndex(self):
from Products.PluginIndexes.interfaces import IPathIndex
from zope.interface.verify import verifyObject
verifyObject(IPathIndex, self._makeOne())
def test_ctor(self):
index = self._makeOne()
self.assertEqual(index.id, 'path')
self.assertEqual(index.operators, ('or', 'and'))
self.assertEqual(index.useOperator, 'or')
self.assertEqual(len(index), 0)
self.assertEqual(index._depth, 0)
self.assertEqual(len(index._index), 0)
self.assertEqual(len(index._unindex), 0)
self.assertEqual(index._length(), 0)
def test_getEntryForObject_miss_no_default(self):
index = self._makeOne()
self.assertEqual(index.getEntryForObject(1234), None)
def test_getEntryForObject_miss_w_default(self):
index = self._makeOne()
default = object()
self.assertTrue(index.getEntryForObject(1234, default) is default)
def test_getEntryForObject_hit(self):
index = self._makeOne()
_populateIndex(index)
self.assertEqual(index.getEntryForObject(1), DUMMIES[1].path)
def test_getIndexSourceNames(self):
index = self._makeOne('foo')
self.assertEqual(list(index.getIndexSourceNames()),
['foo', 'getPhysicalPath'])
def test_index_object_broken_path_raises_TypeError(self):
index = self._makeOne()
doc = Dummy({})
self.assertRaises(TypeError, index.index_object, 1, doc)
def test_index_object_broken_callable(self):
index = self._makeOne()
doc = Dummy(lambda: self.nonesuch)
rc = index.index_object(1, doc)
self.assertEqual(rc, 0)
self.assertEqual(len(index), 0)
self.assertEqual(index._depth, 0)
self.assertEqual(len(index._index), 0)
self.assertEqual(len(index._unindex), 0)
self.assertEqual(index._length(), 0)
def test_index_object_at_root(self):
index = self._makeOne()
doc = Dummy('/xx')
rc = index.index_object(1, doc)
self.assertEqual(len(index), 1)
self.assertEqual(rc, 1)
self.assertEqual(index._depth, 0)
self.assertEqual(len(index._index), 1)
self.assertEqual(list(index._index['xx'][0]), [1])
self.assertEqual(len(index._unindex), 1)
self.assertEqual(index._unindex[1], '/xx')
self.assertEqual(index._length(), 1)
def test_index_object_at_root_callable_attr(self):
index = self._makeOne()
doc = Dummy(lambda: '/xx')
rc = index.index_object(1, doc)
self.assertEqual(len(index), 1)
self.assertEqual(rc, 1)
self.assertEqual(index._depth, 0)
self.assertEqual(len(index._index), 1)
self.assertEqual(list(index._index['xx'][0]), [1])
self.assertEqual(len(index._unindex), 1)
self.assertEqual(index._unindex[1], '/xx')
self.assertEqual(index._length(), 1)
def test_index_object_at_root_no_attr_but_getPhysicalPath(self):
class Other:
def getPhysicalPath(self):
return '/xx'
index = self._makeOne()
doc = Other()
rc = index.index_object(1, doc)
self.assertEqual(rc, 1)
self.assertEqual(len(index), 1)
self.assertEqual(index._depth, 0)
self.assertEqual(len(index._index), 1)
self.assertEqual(list(index._index['xx'][0]), [1])
self.assertEqual(len(index._unindex), 1)
self.assertEqual(index._unindex[1], '/xx')
self.assertEqual(index._length(), 1)
def test_index_object_at_root_attr_as_tuple(self):
index = self._makeOne()
doc = Dummy(('', 'xx'))
rc = index.index_object(1, doc)
self.assertEqual(rc, 1)
self.assertEqual(len(index), 1)
self.assertEqual(index._depth, 0)
self.assertEqual(len(index._index), 1)
self.assertEqual(list(index._index['xx'][0]), [1])
self.assertEqual(len(index._unindex), 1)
self.assertEqual(index._unindex[1], '/xx')
self.assertEqual(index._length(), 1)
def test_index_object_strips_empty_path_elements(self):
index = self._makeOne()
doc = Dummy('////xx//')
rc = index.index_object(1, doc)
self.assertEqual(rc, 1)
self.assertEqual(len(index), 1)
self.assertEqual(index._depth, 0)
self.assertEqual(len(index._index), 1)
self.assertEqual(list(index._index['xx'][0]), [1])
self.assertEqual(len(index._unindex), 1)
self.assertEqual(index._unindex[1], '////xx//')
self.assertEqual(index._length(), 1)
def test_index_object_below_root(self):
index = self._makeOne()
doc = Dummy('/xx/yy/zz')
rc = index.index_object(1, doc)
self.assertEqual(rc, 1)
self.assertEqual(len(index), 1)
self.assertEqual(index._depth, 2)
self.assertEqual(len(index._index), 3)
self.assertEqual(list(index._index['xx'][0]), [1])
self.assertEqual(list(index._index['yy'][1]), [1])
self.assertEqual(list(index._index['zz'][2]), [1])
self.assertEqual(len(index._unindex), 1)
self.assertEqual(index._unindex[1], '/xx/yy/zz')
self.assertEqual(index._length(), 1)
def test_index_object_again(self):
index = self._makeOne()
o = Dummy('/foo/bar')
index.index_object(1234, o)
self.assertEqual(len(index), 1)
self.assertEqual(index.numObjects(), 1)
index.index_object(1234, o)
self.assertEqual(len(index), 1)
self.assertEqual(index.numObjects(), 1)
def test_unindex_object_nonesuch(self):
index = self._makeOne()
index.unindex_object( 1234 ) # nothrow
def test_unindex_object_broken_path(self):
index = self._makeOne()
_populateIndex(index)
index._unindex[1] = "/broken/thing"
index.unindex_object(1) # nothrow
def test_unindex_object_found(self):
index = self._makeOne()
_populateIndex(index)
for k in DUMMIES.keys():
index.unindex_object(k)
self.assertEqual(index.numObjects(), 0)
self.assertEqual(len(index._index), 0)
self.assertEqual(len(index._unindex), 0)
def test__apply_index_no_match_in_query(self):
index = self._makeOne()
self.assertEqual(index._apply_index({'foo': 'xxx'}), None)
def test__apply_index_nonesuch(self):
index = self._makeOne()
res = index._apply_index({'path': 'xxx'})
self.assertEqual(len(res[0]), 0)
self.assertEqual(res[1], ('path',))
def test___apply_index_root_levelO_dict(self):
index = self._makeOne()
_populateIndex(index)
query = {'path': {'query': '/', 'level': 0}}
res = index._apply_index(query)
self.assertEqual(list(res[0].keys()), range(1,19))
def test___apply_index_root_levelO_tuple(self):
index = self._makeOne()
_populateIndex(index)
query = {'path': (('/', 0),)}
res = index._apply_index(query)
self.assertEqual(list(res[0].keys()), range(1,19))
def test__apply_index_simple(self):
index = self._makeOne()
_populateIndex(index)
tests = [
# component, level, expected results
("aa", 0, [1,2,3,4,5,6,7,8,9]),
("aa", 1, [1,2,3,10,11,12] ),
("bb", 0, [10,11,12,13,14,15,16,17,18]),
("bb", 1, [4,5,6,13,14,15]),
("bb/cc", 0, [16,17,18]),
("bb/cc", 1, [6,15]),
("bb/aa", 0, [10,11,12]),
("bb/aa", 1, [4,13]),
("aa/cc", -1, [3,7,8,9,12]),
("bb/bb", -1, [5,13,14,15]),
("18.html", 3, [18]),
("18.html", -1, [18]),
("cc/18.html", -1, [18]),
("cc/18.html", 2, [18]),
]
for comp, level, expected in tests:
for path in [comp, "/"+comp, "/"+comp+"/"]:
# Test with the level passed in as separate parameter
query = {'path': {'query':path, 'level': level}}
res = index._apply_index(query)
self.assertEqual(list(res[0].keys()), expected)
# Test with the level passed in as part of the path parameter
query = {'path': ((path, level),)}
res = index._apply_index(query)
self.assertEqual(list(res[0].keys()), expected)
def test__apply_index_ComplexOrTests(self):
index = self._makeOne()
_populateIndex(index)
tests = [
(['aa','bb'],1,[1,2,3,4,5,6,10,11,12,13,14,15]),
(['aa','bb','xx'],1,[1,2,3,4,5,6,10,11,12,13,14,15]),
([('cc',1),('cc',2)],0,[3,6,7,8,9,12,15,16,17,18]),
]
for lst, level, expected in tests:
query = {'path': {'query': lst, 'level': level, 'operator': 'or'}}
res = index._apply_index(query)
lst = list(res[0].keys())
self.assertEqual(lst, expected)
def test__apply_index_ComplexANDTests(self):
index = self._makeOne()
_populateIndex(index)
tests = [
# Path query (as list or (path, level) tuple), level, expected
(['aa','bb'], 1, []),
([('aa',0), ('bb',1)], 0, [4,5,6]),
([('aa',0), ('cc',2)], 0, [3,6,9]),
]
for lst, level, expected in tests:
query = {'path': {'query': lst, 'level': level, 'operator': 'and'}}
res = index._apply_index(query)
lst = list(res[0].keys())
self.assertEqual(lst, expected)
def test__apply_index_QueryPathReturnedInResult(self):
index = self._makeOne()
index.index_object(1, Dummy("/ff"))
index.index_object(2, Dummy("/ff/gg"))
index.index_object(3, Dummy("/ff/gg/3.html"))
index.index_object(4, Dummy("/ff/gg/4.html"))
res = index._apply_index({'path': {'query': '/ff/gg'}})
lst = list(res[0].keys())
self.assertEqual(lst, [2, 3, 4])
def test_numObjects_empty(self):
index = self._makeOne()
self.assertEqual(index.numObjects(), 0)
def test_numObjects_filled(self):
index = self._makeOne()
_populateIndex(index)
self.assertEqual(index.numObjects(), len(DUMMIES))
def test_indexSize_empty(self):
index = self._makeOne()
self.assertEqual(index.indexSize(), 0)
def test_indexSize_filled(self):
index = self._makeOne()
_populateIndex(index)
self.assertEqual(index.indexSize(), len(DUMMIES))
def test_indexSize_multiple_items_same_path(self):
index = self._makeOne()
doc1 = Dummy('/shared')
doc2 = Dummy('/shared')
index.index_object(1, doc1)
index.index_object(2, doc2)
self.assertEqual(len(index._index), 1)
self.assertEqual(len(index), 2)
self.assertEqual(index.numObjects(), 2)
self.assertEqual(index.indexSize(), 2)
def test_clear(self):
index = self._makeOne()
_populateIndex(index)
index.clear()
self.assertEqual(len(index), 0)
self.assertEqual(index._depth, 0)
self.assertEqual(len(index._index), 0)
self.assertEqual(len(index._unindex), 0)
self.assertEqual(index._length(), 0)
def test_hasUniqueValuesFor_miss(self):
index = self._makeOne()
self.assertFalse(index.hasUniqueValuesFor('miss'))
def test_hasUniqueValuesFor_hit(self):
index = self._makeOne()
self.assertTrue(index.hasUniqueValuesFor('path'))
def test_uniqueValues_empty(self):
index = self._makeOne()
self.assertEqual(len(list(index.uniqueValues())), 0)
def test_uniqueValues_miss(self):
index = self._makeOne('foo')
_populateIndex(index)
self.assertEqual(len(list(index.uniqueValues('bar'))), 0)
def test_uniqueValues_hit(self):
index = self._makeOne('foo')
_populateIndex(index)
self.assertEqual(len(list(index.uniqueValues('foo'))),
len(DUMMIES) + 3)
def test_uniqueValues_hit_w_withLength(self):
index = self._makeOne('foo')
_populateIndex(index)
results = dict(index.uniqueValues('foo', True))
self.assertEqual(len(results), len(DUMMIES) + 3)
for i in range(1, 19):
self.assertEqual(results['%s.html' % i], 1)
self.assertEqual(results['aa'],
len([x for x in DUMMIES.values() if 'aa' in x.path]))
self.assertEqual(results['bb'],
len([x for x in DUMMIES.values() if 'bb' in x.path]))
self.assertEqual(results['cc'],
len([x for x in DUMMIES.values() if 'cc' in x.path]))
def test_keyForDocument_miss(self):
index = self._makeOne()
self.assertEqual(index.keyForDocument(1), None)
def test_keyForDocument_hit(self):
index = self._makeOne()
_populateIndex(index)
self.assertEqual(index.keyForDocument(1), DUMMIES[1].path)
def test_documentToKeyMap_empty(self):
index = self._makeOne()
self.assertEqual(dict(index.documentToKeyMap()), {})
def test_documentToKeyMap_filled(self):
index = self._makeOne()
_populateIndex(index)
self.assertEqual(dict(index.documentToKeyMap()),
dict([(k, v.path) for k, v in DUMMIES.items()]))
def test_insertEntry_empty_depth_0(self):
index = self._makeOne()
index.insertEntry('xx', 123, level=0)
self.assertEqual(index._depth, 0)
self.assertEqual(len(index._index), 1)
self.assertEqual(list(index._index['xx'][0]), [123])
# insertEntry oesn't update the length or the reverse index.
self.assertEqual(len(index), 0)
self.assertEqual(len(index._unindex), 0)
self.assertEqual(index._length(), 0)
def test_insertEntry_empty_depth_1(self):
index = self._makeOne()
index.insertEntry('xx', 123, level=0)
index.insertEntry('yy', 123, level=1)
self.assertEqual(index._depth, 1)
self.assertEqual(len(index._index), 2)
self.assertEqual(list(index._index['xx'][0]), [123])
self.assertEqual(list(index._index['yy'][1]), [123])
def test_insertEntry_multiple(self):
index = self._makeOne()
index.insertEntry('xx', 123, level=0)
index.insertEntry('yy', 123, level=1)
index.insertEntry('aa', 456, level=0)
index.insertEntry('bb', 456, level=1)
index.insertEntry('cc', 456, level=2)
self.assertEqual(index._depth, 2)
self.assertEqual(len(index._index), 5)
self.assertEqual(list(index._index['xx'][0]), [123])
self.assertEqual(list(index._index['yy'][1]), [123])
self.assertEqual(list(index._index['aa'][0]), [456])
self.assertEqual(list(index._index['bb'][1]), [456])
self.assertEqual(list(index._index['cc'][2]), [456])
def test__search_empty_index_string_query(self):
index = self._makeOne()
self.assertEqual(list(index._search('/xxx')), [])
def test__search_empty_index_tuple_query(self):
index = self._makeOne()
self.assertEqual(list(index._search(('/xxx', 0))), [])
def test__search_empty_path(self):
index = self._makeOne()
doc = Dummy('/aa')
index.index_object(1, doc)
self.assertEqual(list(index._search('/')), [1])
def test__search_matching_path(self):
index = self._makeOne()
doc = Dummy('/aa')
index.index_object(1, doc)
self.assertEqual(list(index._search('/aa')), [1])
def test__search_mismatched_path(self):
index = self._makeOne()
doc = Dummy('/aa')
index.index_object(1, doc)
self.assertEqual(list(index._search('/bb')), [])
def test__search_w_level_0(self):
index = self._makeOne()
doc = Dummy('/aa/bb')
index.index_object(1, doc)
self.assertEqual(list(index._search('aa', 0)), [1])
self.assertEqual(list(index._search('aa', 1)), [])
self.assertEqual(list(index._search('bb', 1)), [1])
self.assertEqual(list(index._search('aa/bb', 0)), [1])
self.assertEqual(list(index._search('aa/bb', 1)), [])
def test_suite():
return unittest.TestSuite((
unittest.makeSuite(PathIndexTests),
))
src/Products/PluginIndexes/TopicIndex/FilteredSet.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Filtered set.
"""
from logging import getLogger
import sys
from BTrees.IIBTree import IITreeSet
from Persistence import Persistent
from RestrictedPython.Eval import RestrictionCapableEval
from ZODB.POSException import ConflictError
from zope.interface import implements
from Products.PluginIndexes.interfaces import IFilteredSet
LOG = getLogger('Zope.TopicIndex.FilteredSet')
class FilteredSetBase(Persistent):
# A pre-calculated result list based on an expression.
implements(IFilteredSet)
def __init__(self, id, expr):
self.id = id
self.expr = expr
self.clear()
def clear(self):
self.ids = IITreeSet()
def index_object(self, documentId, obj):
raise RuntimeError,'index_object not defined'
def unindex_object(self,documentId):
try: self.ids.remove(documentId)
except KeyError: pass
def getId(self):
return self.id
def getExpression(self):
# Get the expression.
return self.expr
def getIds(self):
# Get the IDs of all objects for which the expression is True.
return self.ids
def getType(self):
return self.meta_type
def setExpression(self, expr):
# Set the expression.
self.expr = expr
def __repr__(self):
return '%s: (%s) %s' % (self.id,self.expr,map(None,self.ids))
__str__ = __repr__
class PythonFilteredSet(FilteredSetBase):
meta_type = 'PythonFilteredSet'
def index_object(self, documentId, o):
try:
if RestrictionCapableEval(self.expr).eval({'o': o}):
self.ids.insert(documentId)
else:
try:
self.ids.remove(documentId)
except KeyError:
pass
except ConflictError:
raise
except:
LOG.warn('eval() failed Object: %s, expr: %s' %\
(o.getId(),self.expr), exc_info=sys.exc_info())
def factory(f_id, f_type, expr):
""" factory function for FilteredSets """
if f_type=='PythonFilteredSet':
return PythonFilteredSet(f_id, expr)
else:
raise TypeError,'unknown type for FilteredSets: %s' % f_type
src/Products/PluginIndexes/TopicIndex/README.txt deleted 100644 → 0
View file @ 03a5df08
TopicIndex
Reference: http://dev.zope.org/Wikis/DevSite/Proposals/TopicIndexes
A TopicIndex is a container for so-called FilteredSet. A FilteredSet
consists of an expression and a set of internal ZCatalog document
identifiers that represent a pre-calculated result list for performance
reasons. Instead of executing the same query on a ZCatalog multiple times
it is much faster to use a TopicIndex instead.
Building up FilteredSets happens on the fly when objects are cataloged
and uncatalogued. Every indexed object is evaluated against the expressions
of every FilteredSet. An object is added to a FilteredSet if the expression
with the object evaluates to 1. Uncatalogued objects are removed from the
FilteredSet.
Types of FilteredSet
PythonFilteredSet
A PythonFilteredSet evaluates using the eval() function inside the
context of the FilteredSet class. The object to be indexes must
be referenced inside the expression using "o.".
Examples::
"o.meta_type=='DTML Method'"
Queries on TopicIndexes
A TopicIndex is queried in the same way as other ZCatalog Indexes and supports
usage of the 'operator' parameter to specify how to combine search results.
API
The TopicIndex implements the API for pluggable Indexes.
Additionally it provides the following functions to manage FilteredSets
-- addFilteredSet(Id, filterType, expression):
-- Id: unique Id for the FilteredSet
-- filterType: 'PythonFilteredSet'
-- expression: Python expression
-- delFilteredSet(Id):
-- clearFilteredSet(Id):
src/Products/PluginIndexes/TopicIndex/TopicIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Topic index.
"""
from logging import getLogger
from App.special_dtml import DTMLFile
from BTrees.IIBTree import IITreeSet
from BTrees.IIBTree import intersection
from BTrees.IIBTree import union
from BTrees.OOBTree import OOBTree
from OFS.SimpleItem import SimpleItem
from Persistence import Persistent
from zope.interface import implements
from Products.PluginIndexes.common.util import parseIndexRequest
from Products.PluginIndexes.interfaces import IPluggableIndex
from Products.PluginIndexes.interfaces import ITopicIndex
from Products.PluginIndexes.TopicIndex.FilteredSet import factory
_marker = []
LOG = getLogger('Zope.TopicIndex')
class TopicIndex(Persistent, SimpleItem):
"""A TopicIndex maintains a set of FilteredSet objects.
Every FilteredSet object consists of an expression and and IISet with all
Ids of indexed objects that eval with this expression to 1.
"""
implements(ITopicIndex, IPluggableIndex)
meta_type="TopicIndex"
query_options = ('query', 'operator')
manage_options= (
{'label': 'FilteredSets', 'action': 'manage_main'},
)
def __init__(self,id,caller=None):
self.id = id
self.filteredSets = OOBTree()
self.operators = ('or','and')
self.defaultOperator = 'or'
def getId(self):
return self.id
def clear(self):
for fs in self.filteredSets.values():
fs.clear()
def index_object(self, docid, obj ,threshold=100):
""" hook for (Z)Catalog """
for fid, filteredSet in self.filteredSets.items():
filteredSet.index_object(docid,obj)
return 1
def unindex_object(self,docid):
""" hook for (Z)Catalog """
for fs in self.filteredSets.values():
try:
fs.unindex_object(docid)
except KeyError:
LOG.debug('Attempt to unindex document'
' with id %s failed' % docid)
return 1
def numObjects(self):
"""Return the number of indexed objects."""
return "n/a"
def indexSize(self):
"""Return the size of the index in terms of distinct values."""
return "n/a"
def search(self,filter_id):
if self.filteredSets.has_key(filter_id):
return self.filteredSets[filter_id].getIds()
def _apply_index(self, request):
""" hook for (Z)Catalog
'request' -- mapping type (usually {"topic": "..." }
"""
record = parseIndexRequest(request, self.id, self.query_options)
if record.keys is None:
return None
operator = record.get('operator', self.defaultOperator).lower()
if operator == 'or': set_func = union
else: set_func = intersection
res = None
for filter_id in record.keys:
rows = self.search(filter_id)
res = set_func(res,rows)
if res:
return res, (self.id,)
else:
return IITreeSet(), (self.id,)
def uniqueValues(self,name=None, withLength=0):
""" needed to be consistent with the interface """
return self.filteredSets.keys()
def getEntryForObject(self,docid, default=_marker):
""" Takes a document ID and returns all the information we have
on that specific object.
"""
return self.filteredSets.keys()
def addFilteredSet(self, filter_id, typeFilteredSet, expr):
# Add a FilteredSet object.
if self.filteredSets.has_key(filter_id):
raise KeyError,\
'A FilteredSet with this name already exists: %s' % filter_id
self.filteredSets[filter_id] = factory(filter_id,
typeFilteredSet,
expr,
)
def delFilteredSet(self, filter_id):
# Delete the FilteredSet object specified by 'filter_id'.
if not self.filteredSets.has_key(filter_id):
raise KeyError,\
'no such FilteredSet: %s' % filter_id
del self.filteredSets[filter_id]
def clearFilteredSet(self, filter_id):
# Clear the FilteredSet object specified by 'filter_id'.
if not self.filteredSets.has_key(filter_id):
raise KeyError,\
'no such FilteredSet: %s' % filter_id
self.filteredSets[filter_id].clear()
def manage_addFilteredSet(self, filter_id, typeFilteredSet, expr, URL1, \
REQUEST=None,RESPONSE=None):
""" add a new filtered set """
if len(filter_id) == 0: raise RuntimeError,'Length of ID too short'
if len(expr) == 0: raise RuntimeError,'Length of expression too short'
self.addFilteredSet(filter_id, typeFilteredSet, expr)
if RESPONSE:
RESPONSE.redirect(URL1+'/manage_workspace?'
'manage_tabs_message=FilteredSet%20added')
def manage_delFilteredSet(self, filter_ids=[], URL1=None, \
REQUEST=None,RESPONSE=None):
""" delete a list of FilteredSets"""
for filter_id in filter_ids:
self.delFilteredSet(filter_id)
if RESPONSE:
RESPONSE.redirect(URL1+'/manage_workspace?'
'manage_tabs_message=FilteredSet(s)%20deleted')
def manage_saveFilteredSet(self,filter_id, expr, URL1=None,\
REQUEST=None,RESPONSE=None):
""" save expression for a FilteredSet """
self.filteredSets[filter_id].setExpression(expr)
if RESPONSE:
RESPONSE.redirect(URL1+'/manage_workspace?'
'manage_tabs_message=FilteredSet(s)%20updated')
def getIndexSourceNames(self):
""" return names of indexed attributes """
return ('n/a',)
def manage_clearFilteredSet(self, filter_ids=[], URL1=None, \
REQUEST=None,RESPONSE=None):
""" clear a list of FilteredSets"""
for filter_id in filter_ids:
self.clearFilteredSet(filter_id)
if RESPONSE:
RESPONSE.redirect(URL1+'/manage_workspace?'
'manage_tabs_message=FilteredSet(s)%20cleared')
manage = manage_main = DTMLFile('dtml/manageTopicIndex',globals())
manage_main._setName('manage_main')
editFilteredSet = DTMLFile('dtml/editFilteredSet',globals())
manage_addTopicIndexForm = DTMLFile('dtml/addTopicIndex', globals())
def manage_addTopicIndex(self, id, REQUEST=None, RESPONSE=None, URL3=None):
"""Add a TopicIndex"""
return self.manage_addIndex(id, 'TopicIndex', extra=None, \
REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)
src/Products/PluginIndexes/TopicIndex/__init__.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
src/Products/PluginIndexes/TopicIndex/dtml/addTopicIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var "manage_form_title(this(), _,
form_title='Add TopicIndex',
)">
<p class="form-help">
A <em>TopicIndex</em> is a container for so-called <em>FilteredSets</em>
that consist of an expression and a set of internal ZCatalog document
identifiers that fulfill this expression. <em>TopicIndexes</em> are
useful for performance reasons when search queries take too long
and pre-calculated result sets offer a better performance.
</p>
<form action="manage_addTopicIndex" method="post" enctype="multipart/form-data">
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Id
</div>
</td>
<td align="left" valign="top">
<input type="text" name="id" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-optional">
Type
</div>
</td>
<td align="left" valign="top">
TopicIndex
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/TopicIndex/dtml/editFilteredSet.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p>
<dtml-with "filteredSets[filteredSet]">
<form action="manage_saveFilteredSet" method="post" enctype="multipart/form-data">
<input type="hidden" name="filter_id" value="&dtml-getId;" >
<table cellspacing="0" cellpadding="2" border="1" width="90%" align="center">
<tr>
<th colspan="2">Edit FilteredSet</th>
</tr>
<tr>
<th>FilteredSet Id</th>
<td>
&dtml-getId;
</td>
</tr>
<tr>
<th>FilteredSet Type</th>
<td>&dtml-getType;</td>
</tr>
<tr>
<th>FilteredSet Expression</th>
<td>
<textarea name="expr" cols="60" rows="5">&dtml-getExpression;</textarea>
</td>
</tr>
<tr>
<td colspan="2" align="center">
<input class="form-element" type="submit" value=" Save " />
</td>
</tr>
</table>
</form>
</dtml-with>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/TopicIndex/dtml/manageTopicIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<form action="&dtml-URL1;/" method="post" enctype="multipart/form-data">
<table cellspacing="0" cellpadding="2" border="1" width="90%" align="center">
<tr>
<th colspan="5">
Defined FilteredSets
</th>
</tr>
<dtml-if "_.len(filteredSets.values())>0">
<tr>
<th>&nbsp;</th>
<th>FilteredSet Id</th>
<th>FilteredSet Type</th>
<th>Expression</th>
<th># entries</th>
</tr>
<dtml-in expr="filteredSets.values()">
<dtml-call "REQUEST.set('fs',_['sequence-item'])">
<tr>
<td align="center">
<input type="checkbox" name="filter_ids:list" value="<dtml-var "fs.getId()" html_quote>">
</td>
<td align="center" valign="top">
<div class="form-label">
<a href="editFilteredSet?filteredSet=&dtml-id;">&dtml-getId; </a>
</div>
</td>
<td align="center" valign="top">
<div class="form-label">
&dtml-getType;
</div>
</td>
<td align="left" valign="top">
<div class="form-label">
&dtml-getExpression;
</div>
</td>
<td align="center" valign="top">
<div class="form-label">
<dtml-var "_.len(fs.getIds())">
</div>
</td>
</tr>
</dtml-in>
<tr>
<td colspan="5" align="center">
<input class="form-element" type="submit" name="manage_delFilteredSet:method"
value=" Remove " />
<input class="form-element" type="submit" name="manage_clearFilteredSet:method"
value=" Clear " />
</td>
</tr>
<dtml-else>
<tr>
<td colspan="5" align="center">
<em>no FilteredSets defined </em>
</td>
</tr>
</dtml-if>
</table>
</form>
<hr>
<form action="manage_addFilteredSet" method="post" enctype="multipart/form-data">
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Id for FilteredSet
</div>
</td>
<td align="left" valign="top">
<input type="text" name="filter_id" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-label">
Type of FilteredSet
</div>
</td>
<td align="left" valign="top">
<select name="typeFilteredSet">
<option value="PythonFilteredSet">PythonFilteredSet
<dtml-comment>
<option value="AttributeFilteredSet">AttributeFilteredSet
</dtml-comment>
</select>
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-label">
Expression
</div>
</td>
<td align="left" valign="top">
<textarea type="text" name="expr" cols="60" rows="5"></textarea>
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/TopicIndex/tests/__init__.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
# This file is needed to make this a package.
src/Products/PluginIndexes/TopicIndex/tests/testTopicIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""TopicIndex unit tests.
"""
import unittest
import Zope2
Zope2.startup()
from Products.PluginIndexes.TopicIndex.TopicIndex import TopicIndex
class Obj:
def __init__(self,id,meta_type=''):
self.id = id
self.meta_type = meta_type
def getId(self): return self.id
def getPhysicalPath(self): return self.id
class TestBase(unittest.TestCase):
def _searchAnd(self,query,expected):
return self._search(query,'and',expected)
def _searchOr(self,query,expected):
return self._search(query,'or',expected)
def _search(self,query,operator,expected):
res = self.TI._apply_index({'topic':{'query':query,'operator':operator}})
rows = list(res[0].keys())
rows.sort()
expected.sort()
self.assertEqual(rows,expected,query)
return rows
class TestTopicIndex(TestBase):
def setUp(self):
self.TI = TopicIndex("topic")
self.TI.addFilteredSet("doc1","PythonFilteredSet","o.meta_type=='doc1'")
self.TI.addFilteredSet("doc2","PythonFilteredSet","o.meta_type=='doc2'")
self.TI.index_object(0 , Obj('0',))
self.TI.index_object(1 , Obj('1','doc1'))
self.TI.index_object(2 , Obj('2','doc1'))
self.TI.index_object(3 , Obj('3','doc2'))
self.TI.index_object(4 , Obj('4','doc2'))
self.TI.index_object(5 , Obj('5','doc3'))
self.TI.index_object(6 , Obj('6','doc3'))
def test_interfaces(self):
from Products.PluginIndexes.interfaces import ITopicIndex
from Products.PluginIndexes.interfaces import IPluggableIndex
from zope.interface.verify import verifyClass
verifyClass(ITopicIndex, TopicIndex)
verifyClass(IPluggableIndex, TopicIndex)
def testOr(self):
self._searchOr('doc1',[1,2])
self._searchOr(['doc1'],[1,2])
self._searchOr('doc2',[3,4]),
self._searchOr(['doc2'],[3,4])
self._searchOr(['doc1','doc2'], [1,2,3,4])
def testAnd(self):
self._searchAnd('doc1',[1,2])
self._searchAnd(['doc1'],[1,2])
self._searchAnd('doc2',[3,4])
self._searchAnd(['doc2'],[3,4])
self._searchAnd(['doc1','doc2'],[])
def testRemoval(self):
self.TI.index_object(1, Obj('1','doc2'))
self._searchOr('doc1',[2])
self._searchOr('doc2', [1,3,4])
def test_suite():
return unittest.TestSuite((
unittest.makeSuite(TestTopicIndex),
))
src/Products/PluginIndexes/__init__.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
def initialize(context):
from Products.PluginIndexes.FieldIndex.FieldIndex import FieldIndex
from Products.PluginIndexes.FieldIndex.FieldIndex \
import manage_addFieldIndex
from Products.PluginIndexes.FieldIndex.FieldIndex \
import manage_addFieldIndexForm
context.registerClass(FieldIndex,
permission='Add Pluggable Index',
constructors=(manage_addFieldIndexForm,
manage_addFieldIndex),
icon='www/index.gif',
visibility=None,
)
from Products.PluginIndexes.KeywordIndex.KeywordIndex import KeywordIndex
from Products.PluginIndexes.KeywordIndex.KeywordIndex \
import manage_addKeywordIndex
from Products.PluginIndexes.KeywordIndex.KeywordIndex \
import manage_addKeywordIndexForm
context.registerClass(KeywordIndex,
permission='Add Pluggable Index',
constructors=(manage_addKeywordIndexForm,
manage_addKeywordIndex),
icon='www/index.gif',
visibility=None,
)
from Products.PluginIndexes.TopicIndex.TopicIndex import TopicIndex
from Products.PluginIndexes.TopicIndex.TopicIndex \
import manage_addTopicIndex
from Products.PluginIndexes.TopicIndex.TopicIndex \
import manage_addTopicIndexForm
context.registerClass(TopicIndex,
permission='Add Pluggable Index',
constructors=(manage_addTopicIndexForm,
manage_addTopicIndex),
icon='www/index.gif',
visibility=None,
)
from Products.PluginIndexes.DateIndex.DateIndex import DateIndex
from Products.PluginIndexes.DateIndex.DateIndex \
import manage_addDateIndex
from Products.PluginIndexes.DateIndex.DateIndex \
import manage_addDateIndexForm
context.registerClass(DateIndex,
permission='Add Pluggable Index',
constructors=(manage_addDateIndexForm,
manage_addDateIndex),
icon='www/index.gif',
visibility=None,
)
from Products.PluginIndexes.DateRangeIndex.DateRangeIndex \
import DateRangeIndex
from Products.PluginIndexes.DateRangeIndex.DateRangeIndex \
import manage_addDateRangeIndex
from Products.PluginIndexes.DateRangeIndex.DateRangeIndex \
import manage_addDateRangeIndexForm
context.registerClass(DateRangeIndex,
permission='Add Pluggable Index',
constructors=(manage_addDateRangeIndexForm,
manage_addDateRangeIndex),
icon='www/index.gif',
visibility=None,
)
from Products.PluginIndexes.PathIndex.PathIndex import PathIndex
from Products.PluginIndexes.PathIndex.PathIndex \
import manage_addPathIndex
from Products.PluginIndexes.PathIndex.PathIndex \
import manage_addPathIndexForm
context.registerClass(PathIndex,
permission='Add Pluggable Index',
constructors=(manage_addPathIndexForm,
manage_addPathIndex),
icon='www/index.gif',
visibility=None,
)
from Products.PluginIndexes.BooleanIndex.BooleanIndex import BooleanIndex
from Products.PluginIndexes.BooleanIndex.BooleanIndex import \
manage_addBooleanIndex
from Products.PluginIndexes.BooleanIndex.BooleanIndex import \
manage_addBooleanIndexForm
context.registerClass(BooleanIndex,
permission='Add Pluggable Index',
constructors=(manage_addBooleanIndexForm,
manage_addBooleanIndex),
icon='www/index.gif',
visibility=None,
)
src/Products/PluginIndexes/common/ResultList.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
from BTrees.IIBTree import difference
from BTrees.IIBTree import IIBucket
from BTrees.IIBTree import weightedIntersection
from BTrees.IIBTree import weightedUnion
from BTrees.OOBTree import OOSet
from BTrees.OOBTree import union
class ResultList:
def __init__(self, d, words, index):
self._index = index
if type(words) is not OOSet:
words=OOSet(words)
self._words = words
if (type(d) is tuple):
d = IIBucket((d,))
elif type(d) is not IIBucket:
d = IIBucket(d)
self._dict=d
self.__getitem__=d.__getitem__
try: self.__nonzero__=d.__nonzero__
except: pass
self.get=d.get
def __nonzero__(self):
return not not self._dict
def bucket(self):
return self._dict
def keys(self):
return self._dict.keys()
def has_key(self, key):
return self._dict.has_key(key)
def items(self):
return self._dict.items()
def __and__(self, x):
return self.__class__(
weightedIntersection(self._dict, x._dict)[1],
union(self._words, x._words),
self._index,
)
def and_not(self, x):
return self.__class__(
difference(self._dict, x._dict),
self._words,
self._index,
)
def __or__(self, x):
return self.__class__(
weightedUnion(self._dict, x._dict)[1],
union(self._words, x._words),
self._index,
)
# return self.__class__(result, self._words+x._words, self._index)
def near(self, x):
result = IIBucket()
dict = self._dict
xdict = x._dict
xhas = xdict.has_key
positions = self._index.positions
for id, score in dict.items():
if not xhas(id): continue
p=(map(lambda i: (i,0), positions(id,self._words))+
map(lambda i: (i,1), positions(id,x._words)))
p.sort()
d = lp = 9999
li = None
lsrc = None
for i,src in p:
if i is not li and src is not lsrc and li is not None:
d = min(d,i-li)
li = i
lsrc = src
if d==lp: score = min(score,xdict[id]) # synonyms
else: score = (score+xdict[id])/d
result[id] = score
return self.__class__(
result, union(self._words, x._words), self._index)
src/Products/PluginIndexes/common/UnIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Base for bi-directional indexes.
"""
from cgi import escape
from logging import getLogger
import sys
from BTrees.IIBTree import intersection
from BTrees.IIBTree import IITreeSet
from BTrees.IIBTree import IISet
from BTrees.IIBTree import multiunion
from BTrees.IOBTree import IOBTree
from BTrees.Length import Length
from BTrees.OOBTree import OOBTree
from OFS.SimpleItem import SimpleItem
from ZODB.POSException import ConflictError
from zope.interface import implements
from Products.PluginIndexes.common import safe_callable
from Products.PluginIndexes.common.util import parseIndexRequest
from Products.PluginIndexes.interfaces import ILimitedResultIndex
from Products.PluginIndexes.interfaces import ISortIndex
from Products.PluginIndexes.interfaces import IUniqueValueIndex
_marker = []
LOG = getLogger('Zope.UnIndex')
class UnIndex(SimpleItem):
"""Simple forward and reverse index.
"""
implements(ILimitedResultIndex, IUniqueValueIndex, ISortIndex)
def __init__(
self, id, ignore_ex=None, call_methods=None, extra=None, caller=None):
"""Create an unindex
UnIndexes are indexes that contain two index components, the
forward index (like plain index objects) and an inverted
index. The inverted index is so that objects can be unindexed
even when the old value of the object is not known.
e.g.
self._index = {datum:[documentId1, documentId2]}
self._unindex = {documentId:datum}
If any item in self._index has a length-one value, the value is an
integer, and not a set. There are special cases in the code to deal
with this.
The arguments are:
'id' -- the name of the item attribute to index. This is
either an attribute name or a record key.
'ignore_ex' -- should be set to true if you want the index
to ignore exceptions raised while indexing instead of
propagating them.
'call_methods' -- should be set to true if you want the index
to call the attribute 'id' (note: 'id' should be callable!)
You will also need to pass in an object in the index and
uninded methods for this to work.
'extra' -- a mapping object that keeps additional
index-related parameters - subitem 'indexed_attrs'
can be string with comma separated attribute names or
a list
'caller' -- reference to the calling object (usually
a (Z)Catalog instance
"""
def _get(o, k, default):
""" return a value for a given key of a dict/record 'o' """
if isinstance(o, dict):
return o.get(k, default)
else:
return getattr(o, k, default)
self.id = id
self.ignore_ex=ignore_ex # currently unimplimented
self.call_methods=call_methods
self.operators = ('or', 'and')
self.useOperator = 'or'
# allow index to index multiple attributes
ia = _get(extra, 'indexed_attrs', id)
if isinstance(ia, str):
self.indexed_attrs = ia.split(',')
else:
self.indexed_attrs = list(ia)
self.indexed_attrs = [ attr.strip()
for attr in self.indexed_attrs if attr ]
if not self.indexed_attrs:
self.indexed_attrs = [id]
self._length = Length()
self.clear()
def __len__(self):
return self._length()
def getId(self):
return self.id
def clear(self):
self._length = Length()
self._index = OOBTree()
self._unindex = IOBTree()
def __nonzero__(self):
return not not self._unindex
def histogram(self):
"""Return a mapping which provides a histogram of the number of
elements found at each point in the index.
"""
histogram = {}
for item in self._index.items():
if isinstance(item,int):
entry = 1 # "set" length is 1
else:
key, value = item
entry = len(value)
histogram[entry] = histogram.get(entry, 0) + 1
return histogram
def referencedObjects(self):
"""Generate a list of IDs for which we have referenced objects."""
return self._unindex.keys()
def getEntryForObject(self, documentId, default=_marker):
"""Takes a document ID and returns all the information we have
on that specific object.
"""
if default is _marker:
return self._unindex.get(documentId)
else:
return self._unindex.get(documentId, default)
def removeForwardIndexEntry(self, entry, documentId):
"""Take the entry provided and remove any reference to documentId
in its entry in the index.
"""
indexRow = self._index.get(entry, _marker)
if indexRow is not _marker:
try:
indexRow.remove(documentId)
if not indexRow:
del self._index[entry]
self._length.change(-1)
except ConflictError:
raise
except AttributeError:
# index row is an int
try:
del self._index[entry]
except KeyError:
# XXX swallow KeyError because it was probably
# removed and then _length AttributeError raised
pass
if isinstance(self.__len__, Length):
self._length = self.__len__
del self.__len__
self._length.change(-1)
except:
LOG.error('%s: unindex_object could not remove '
'documentId %s from index %s. This '
'should not happen.' % (self.__class__.__name__,
str(documentId), str(self.id)),
exc_info=sys.exc_info())
else:
LOG.error('%s: unindex_object tried to retrieve set %s '
'from index %s but couldn\'t. This '
'should not happen.' % (self.__class__.__name__,
repr(entry), str(self.id)))
def insertForwardIndexEntry(self, entry, documentId):
"""Take the entry provided and put it in the correct place
in the forward index.
This will also deal with creating the entire row if necessary.
"""
indexRow = self._index.get(entry, _marker)
# Make sure there's actually a row there already. If not, create
# a set and stuff it in first.
if indexRow is _marker:
# We always use a set to avoid getting conflict errors on
# multiple threads adding a new row at the same time
self._index[entry] = IITreeSet((documentId, ))
self._length.change(1)
else:
try:
indexRow.insert(documentId)
except AttributeError:
# Inline migration: index row with one element was an int at
# first (before Zope 2.13).
indexRow = IITreeSet((indexRow, documentId))
self._index[entry] = indexRow
def index_object(self, documentId, obj, threshold=None):
""" wrapper to handle indexing of multiple attributes """
fields = self.getIndexSourceNames()
res = 0
for attr in fields:
res += self._index_object(documentId, obj, threshold, attr)
return res > 0
def _index_object(self, documentId, obj, threshold=None, attr=''):
""" index and object 'obj' with integer id 'documentId'"""
returnStatus = 0
# First we need to see if there's anything interesting to look at
datum = self._get_object_datum(obj, attr)
# We don't want to do anything that we don't have to here, so we'll
# check to see if the new and existing information is the same.
oldDatum = self._unindex.get(documentId, _marker)
if datum != oldDatum:
if oldDatum is not _marker:
self.removeForwardIndexEntry(oldDatum, documentId)
if datum is _marker:
try:
del self._unindex[documentId]
except ConflictError:
raise
except:
LOG.error('Should not happen: oldDatum was there, now its not,'
'for document with id %s' % documentId)
if datum is not _marker:
self.insertForwardIndexEntry(datum, documentId)
self._unindex[documentId] = datum
returnStatus = 1
return returnStatus
def _get_object_datum(self,obj, attr):
# self.id is the name of the index, which is also the name of the
# attribute we're interested in. If the attribute is callable,
# we'll do so.
try:
datum = getattr(obj, attr)
if safe_callable(datum):
datum = datum()
except (AttributeError, TypeError):
datum = _marker
return datum
def numObjects(self):
"""Return the number of indexed objects."""
return len(self._unindex)
def indexSize(self):
"""Return the size of the index in terms of distinct values."""
return len(self)
def unindex_object(self, documentId):
""" Unindex the object with integer id 'documentId' and don't
raise an exception if we fail
"""
unindexRecord = self._unindex.get(documentId, _marker)
if unindexRecord is _marker:
return None
self.removeForwardIndexEntry(unindexRecord, documentId)
try:
del self._unindex[documentId]
except ConflictError:
raise
except:
LOG.debug('Attempt to unindex nonexistent document'
' with id %s' % documentId,exc_info=True)
def _apply_index(self, request, resultset=None):
"""Apply the index to query parameters given in the request arg.
The request argument should be a mapping object.
If the request does not have a key which matches the "id" of
the index instance, then None is returned.
If the request *does* have a key which matches the "id" of
the index instance, one of a few things can happen:
- if the value is a blank string, None is returned (in
order to support requests from web forms where
you can't tell a blank string from empty).
- if the value is a nonblank string, turn the value into
a single-element sequence, and proceed.
- if the value is a sequence, return a union search.
- If the value is a dict and contains a key of the form
'<index>_operator' this overrides the default method
('or') to combine search results. Valid values are "or"
and "and".
If None is not returned as a result of the abovementioned
constraints, two objects are returned. The first object is a
ResultSet containing the record numbers of the matching
records. The second object is a tuple containing the names of
all data fields used.
FAQ answer: to search a Field Index for documents that
have a blank string as their value, wrap the request value
up in a tuple ala: request = {'id':('',)}
"""
record = parseIndexRequest(request, self.id, self.query_options)
if record.keys is None:
return None
index = self._index
r = None
opr = None
# experimental code for specifing the operator
operator = record.get('operator',self.useOperator)
if not operator in self.operators :
raise RuntimeError("operator not valid: %s" % escape(operator))
# Range parameter
range_parm = record.get('range',None)
if range_parm:
opr = "range"
opr_args = []
if range_parm.find("min")>-1:
opr_args.append("min")
if range_parm.find("max")>-1:
opr_args.append("max")
if record.get('usage',None):
# see if any usage params are sent to field
opr = record.usage.lower().split(':')
opr, opr_args=opr[0], opr[1:]
if opr=="range": # range search
if 'min' in opr_args: lo = min(record.keys)
else: lo = None
if 'max' in opr_args: hi = max(record.keys)
else: hi = None
if hi:
setlist = index.values(lo,hi)
else:
setlist = index.values(lo)
# If we only use one key, intersect and return immediately
if len(setlist) == 1:
result = setlist[0]
if isinstance(result, int):
result = IISet((result,))
return result, (self.id,)
if operator == 'or':
tmp = []
for s in setlist:
if isinstance(s, int):
s = IISet((s,))
tmp.append(s)
r = multiunion(tmp)
else:
# For intersection, sort with smallest data set first
tmp = []
for s in setlist:
if isinstance(s, int):
s = IISet((s,))
tmp.append(s)
if len(tmp) > 2:
setlist = sorted(tmp, key=len)
else:
setlist = tmp
r = resultset
for s in setlist:
# the result is bound by the resultset
r = intersection(r, s)
else: # not a range search
# Filter duplicates
setlist = []
for k in record.keys:
s = index.get(k, None)
# If None, try to bail early
if s is None:
if operator == 'or':
# If union, we can't possibly get a bigger result
continue
# If intersection, we can't possibly get a smaller result
return IISet(), (self.id,)
elif isinstance(s, int):
s = IISet((s,))
setlist.append(s)
# If we only use one key return immediately
if len(setlist) == 1:
result = setlist[0]
if isinstance(result, int):
result = IISet((result,))
return result, (self.id,)
if operator == 'or':
# If we already get a small result set passed in, intersecting
# the various indexes with it and doing the union later is
# faster than creating a multiunion first.
if resultset is not None and len(resultset) < 200:
smalllist = []
for s in setlist:
smalllist.append(intersection(resultset, s))
r = multiunion(smalllist)
else:
r = multiunion(setlist)
else:
# For intersection, sort with smallest data set first
if len(setlist) > 2:
setlist = sorted(setlist, key=len)
r = resultset
for s in setlist:
r = intersection(r, s)
if isinstance(r, int):
r = IISet((r, ))
if r is None:
return IISet(), (self.id,)
else:
return r, (self.id,)
def hasUniqueValuesFor(self, name):
"""has unique values for column name"""
if name == self.id:
return 1
else:
return 0
def getIndexSourceNames(self):
""" return sequence of indexed attributes """
# BBB: older indexes didn't have 'indexed_attrs'
return getattr(self, 'indexed_attrs', [self.id])
def uniqueValues(self, name=None, withLengths=0):
"""returns the unique values for name
if withLengths is true, returns a sequence of
tuples of (value, length)
"""
if name is None:
name = self.id
elif name != self.id:
return []
if not withLengths:
return tuple(self._index.keys())
else:
rl=[]
for i in self._index.keys():
set = self._index[i]
if isinstance(set, int):
l = 1
else:
l = len(set)
rl.append((i, l))
return tuple(rl)
def keyForDocument(self, id):
# This method is superceded by documentToKeyMap
return self._unindex[id]
def documentToKeyMap(self):
return self._unindex
def items(self):
items = []
for k,v in self._index.items():
if isinstance(v, int):
v = IISet((v,))
items.append((k, v))
return items
src/Products/PluginIndexes/common/__init__.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
#############################################################################
# This code is duplicated here from Products/ZCatalog/Catalog.py to avoid a
# unnecessary dependency on ZCatalog.
import types
try:
from DocumentTemplate.cDocumentTemplate import safe_callable
except ImportError:
def safe_callable(ob):
# Works with ExtensionClasses and Acquisition.
if hasattr(ob, '__class__'):
if hasattr(ob, '__call__'):
return 1
else:
return isinstance(ob, types.ClassType)
else:
return callable(ob)
src/Products/PluginIndexes/common/randid.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
#############################################################################
import random
def randid(randint=random.randint, choice=random.choice, signs=(-1,1)):
return choice(signs)*randint(1,2000000000)
del random
src/Products/PluginIndexes/common/tests/__init__.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
#############################################################################
src/Products/PluginIndexes/common/tests/test_UnIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
#############################################################################
""" Tests for common UnIndex features.
"""
import unittest
class UnIndexTests(unittest.TestCase):
def _getTargetClass(self):
from Products.PluginIndexes.common.UnIndex import UnIndex
return UnIndex
def _makeOne(self, *args, **kw):
return self._getTargetClass()(*args, **kw)
def _makeConflicted(self):
from ZODB.POSException import ConflictError
class Conflicted:
def __str__(self):
return 'Conflicted'
__repr__ = __str__
def __getattr__(self, id, default=object()):
raise ConflictError, 'testing'
return Conflicted()
def test_empty(self):
unindex = self._makeOne(id='empty')
self.assertEqual(unindex.indexed_attrs, ['empty'])
def test_removeForwardIndexEntry_with_ConflictError(self):
from ZODB.POSException import ConflictError
unindex = self._makeOne(id='conflicted')
unindex._index['conflicts'] = self._makeConflicted()
self.assertRaises(ConflictError, unindex.removeForwardIndexEntry,
'conflicts', 42)
def test_get_object_datum(self):
from Products.PluginIndexes.common.UnIndex import _marker
idx = self._makeOne('interesting')
dummy = object()
self.assertEquals(idx._get_object_datum(dummy, 'interesting'), _marker)
class DummyContent2(object):
interesting = 'GOT IT'
dummy = DummyContent2()
self.assertEquals(idx._get_object_datum(dummy, 'interesting'), 'GOT IT')
class DummyContent3(object):
exc = None
def interesting(self):
if self.exc:
raise self.exc
return 'GOT IT'
dummy = DummyContent3()
self.assertEquals(idx._get_object_datum(dummy, 'interesting'), 'GOT IT')
dummy.exc = AttributeError
self.assertEquals(idx._get_object_datum(dummy, 'interesting'), _marker)
dummy.exc = TypeError
self.assertEquals(idx._get_object_datum(dummy, 'interesting'), _marker)
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(UnIndexTests))
return suite
src/Products/PluginIndexes/common/tests/test_util.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2007 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Unit tests for util module.
"""
import unittest
from ZPublisher.HTTPRequest import record as Record
class parseIndexRequestTests(unittest.TestCase):
def _getTargetClass(self):
from Products.PluginIndexes.common.util import parseIndexRequest
return parseIndexRequest
def _makeOne(self, *args, **kw):
return self._getTargetClass()(*args, **kw)
def test_get_record(self):
record = Record()
record.query = 'foo'
record.level = 0
record.operator = 'and'
request = {'path': record}
parser = self._makeOne(request, 'path', ('query','level','operator'))
self.assertEqual(parser.get('keys'), ['foo'])
self.assertEqual(parser.get('level'), 0)
self.assertEqual(parser.get('operator'), 'and')
def test_get_dict(self):
request = {'path': {'query': 'foo', 'level': 0, 'operator': 'and'}}
parser = self._makeOne(request, 'path', ('query','level','operator'))
self.assertEqual(parser.get('keys'), ['foo'])
self.assertEqual(parser.get('level'), 0)
self.assertEqual(parser.get('operator'), 'and')
def test_get_string(self):
request = {'path': 'foo', 'path_level': 0, 'path_operator': 'and'}
parser = self._makeOne(request, 'path', ('query','level','operator'))
self.assertEqual(parser.get('keys'), ['foo'])
self.assertEqual(parser.get('level'), 0)
self.assertEqual(parser.get('operator'), 'and')
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(parseIndexRequestTests))
return suite
src/Products/PluginIndexes/common/util.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""PluginIndexes utils.
"""
from types import InstanceType
from DateTime.DateTime import DateTime
class IndexRequestParseError(Exception):
pass
class parseIndexRequest:
"""
This class provides functionality to hide the internals of a request
send from the Catalog/ZCatalog to an index._apply_index() method.
The class understands the following type of parameters:
- old-style parameters where the query for an index as value inside
the request directory where the index name is the name of the key.
- dictionary-style parameters specify a query for an index as
an entry in the request dictionary where the key corresponds to the
name of the index and the key is a dictionary with the parameters
passed to the index.
Allowed keys of the parameter dictionary:
'query' - contains the query (either string, list or tuple) (required)
other parameters depend on the the index
- record-style parameters specify a query for an index as instance of the
Record class. This happens usually when parameters from a web form use
the "record" type e.g. <input type="text" name="path.query:record:string">.
All restrictions of the dictionary-style parameters apply to the record-style
parameters
"""
ParserException = IndexRequestParseError
def __init__(self, request, iid, options=[]):
""" parse a request from the ZPublisher and return a uniform
datastructure back to the _apply_index() method of the index
request -- the request dictionary send from the ZPublisher
iid -- Id of index
options -- a list of options the index is interested in
"""
self.id = iid
if not request.has_key(iid):
self.keys = None
return
param = request[iid]
keys = None
if isinstance(param, InstanceType) and not isinstance(param, DateTime):
""" query is of type record """
record = param
if not hasattr(record, 'query'):
raise self.ParserException(
"record for '%s' *must* contain a "
"'query' attribute" % self.id)
keys = record.query
if isinstance(keys, str):
keys = [keys.strip()]
for op in options:
if op == "query": continue
if hasattr(record, op):
setattr(self, op, getattr(record, op))
elif isinstance(param, dict):
""" query is a dictionary containing all parameters """
query = param.get("query", ())
if isinstance(query, (tuple, list)):
keys = query
else:
keys = [ query ]
for op in options:
if op == "query": continue
if param.has_key(op):
setattr(self, op, param[op])
else:
""" query is tuple, list, string, number, or something else """
if isinstance(param, (tuple, list)):
keys = param
else:
keys = [param]
for op in options:
field = iid + "_" + op
if request.has_key(field):
setattr(self, op, request[field])
self.keys = keys
def get(self, k, default_v=None):
if hasattr(self, k):
v = getattr(self, k)
if v != '':
return v
return default_v
src/Products/PluginIndexes/dtml/browseIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<dtml-call "REQUEST.RESPONSE.setHeader('Content-Type', 'text/html; charset=UTF-8')" >
<p class="form-text">
The index "&dtml-getId;" contains <dtml-var items fmt=collection-length thousands_commas> distinct values
</p>
<dtml-let size="20"> <!-- batch size -->
<div class="form-text">
<dtml-in items previous size=size start=query_start >
<a href="&dtml-URL;?query_start=&dtml-previous-sequence-start-number;">
[Previous <dtml-var previous-sequence-size> entries]
</a>
</dtml-in>
<dtml-in items next size=size start=query_start >
<a href="&dtml-URL;?query_start=&dtml-next-sequence-start-number;">
[Next <dtml-var next-sequence-size> entries]
</a>
</dtml-in>
</div>
<table border="1" align="center" width="100%" class="form-help">
<dtml-in items start=query_start size=size>
<tr>
<td>
<dtml-if "meta_type in ('DateIndex',)">
<dtml-comment><!--
DateIndexes store dates packed into an integer, unpack
into year, month, day, hour and minute, no seconds and UTC.
--></dtml-comment>
<dtml-var "DateTime((_['sequence-key'] / 535680),
(_['sequence-key'] / 44640 ) % 12,
(_['sequence-key'] / 1440 ) % 31,
(_['sequence-key'] / 60 ) % 24,
(_['sequence-key'] ) % 60,
0, 'UTC')">
<dtml-else>
&dtml-sequence-key;
</dtml-if>
</td>
<td>
<ul>
<dtml-let v="_['sequence-item']">
<dtml-if "isinstance(v, int)">
<li><a href="<dtml-var "getpath(v)">"<dtml-var "getpath(v)"></a></li>
<dtml-else>
<dtml-in "v.keys()">
<li> <a href="<dtml-var "getpath(_['sequence-item'])">"><dtml-var "getpath(_['sequence-item'])"></a></li>
</dtml-in>
</dtml-if>
</dtml-let>
</ul>
</td>
</tr>
</dtml-in>
</table>
</dtml-let>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/interfaces.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2005 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""PluginIndexes interfaces.
"""
from zope.interface import Interface
from zope.schema import Bool
class IPluggableIndex(Interface):
def getId():
"""Return Id of index."""
def getEntryForObject(documentId, default=None):
"""Get all information contained for 'documentId'."""
def getIndexSourceNames():
"""Get a sequence of attribute names that are indexed by the index.
"""
def index_object(documentId, obj, threshold=None):
"""Index an object.
- ``documentId`` is the integer ID of the document.
- ``obj`` is the object to be indexed.
- ``threshold`` is the number of words to process between committing
subtransactions. If None, subtransactions are disabled.
For each name in ``getIndexSourceNames``, try to get the named
attribute from ``obj``.
- If the object does not have the attribute, do not add it to the
index for that name.
- If the attribute is a callable, call it to get the value. If
calling it raises an AttributeError, do not add it to the index.
for that name.
"""
def unindex_object(documentId):
"""Remove the documentId from the index."""
def _apply_index(request):
"""Apply the index to query parameters given in 'request'.
The argument should be a mapping object.
If the request does not contain the needed parameters, then
None is returned.
If the request contains a parameter with the name of the
column and this parameter is either a Record or a class
instance then it is assumed that the parameters of this index
are passed as attribute (Note: this is the recommended way to
pass parameters since Zope 2.4)
Otherwise two objects are returned. The first object is a
ResultSet containing the record numbers of the matching
records. The second object is a tuple containing the names of
all data fields used.
"""
def numObjects():
"""Return the number of indexed objects."""
def indexSize():
"""Return the size of the index in terms of distinct values."""
def clear():
"""Empty the index"""
class ILimitedResultIndex(IPluggableIndex):
def _apply_index(request, resultset=None):
"""Same as IPluggableIndex' _apply_index method. The additional
resultset argument contains the resultset, as already calculated by
ZCatalog's search method.
"""
class IUniqueValueIndex(IPluggableIndex):
"""An index which can return lists of unique values contained in it"""
def hasUniqueValuesFor(name):
"""Return true if the index can return the unique values for name"""
def uniqueValues(name=None, withLengths=0):
"""Return the unique values for name.
If 'withLengths' is true, returns a sequence of tuples of
(value, length)."""
class ISortIndex(IPluggableIndex):
"""An index which may be used to sort a set of document ids"""
def keyForDocument(documentId):
"""Return the sort key that cooresponds to the specified document id
This method is no longer used by ZCatalog, but is left for backwards
compatibility."""
def documentToKeyMap():
"""Return an object that supports __getitem__ and may be used to
quickly lookup the sort key given a document id"""
class IDateIndex(Interface):
"""Index for dates.
"""
index_naive_time_as_local = Bool(title=u'Index naive time as local?')
class IDateRangeIndex(Interface):
"""Index for date ranges, such as the "effective-expiration" range in CMF.
Any object may return None for either the start or the end date: for the
start date, this should be the logical equivalent of "since the beginning
of time"; for the end date, "until the end of time".
Therefore, divide the space of indexed objects into four containers:
- Objects which always match (i.e., they returned None for both);
- Objects which match after a given time (i.e., they returned None for the
end date);
- Objects which match until a given time (i.e., they returned None for the
start date);
- Objects which match only during a specific interval.
"""
def getSinceField():
"""Get the name of the attribute indexed as start date.
"""
def getUntilField():
"""Get the name of the attribute indexed as end date.
"""
class IPathIndex(Interface):
"""Index for paths returned by getPhysicalPath.
A path index stores all path components of the physical path of an object.
Internal datastructure:
- a physical path of an object is split into its components
- every component is kept as a key of a OOBTree in self._indexes
- the value is a mapping 'level of the path component' to
'all docids with this path component on this level'
"""
def insertEntry(comp, id, level):
""" Insert an entry.
This method is intended for use by subclasses: it is not
a normal API for the index.
'comp' is an individual path component
'id' is the docid
.level'is the level of the component inside the path
"""
class IFilteredSet(Interface):
"""A pre-calculated result list based on an expression.
"""
def getExpression():
"""Get the expression.
"""
def getIds():
"""Get the IDs of all objects for which the expression is True.
"""
def setExpression(expr):
"""Set the expression.
"""
class ITopicIndex(Interface):
"""A TopicIndex maintains a set of FilteredSet objects.
Every FilteredSet object consists of an expression and and IISet with all
Ids of indexed objects that eval with this expression to 1.
"""
def addFilteredSet(filter_id, typeFilteredSet, expr):
"""Add a FilteredSet object.
"""
def delFilteredSet(filter_id):
"""Delete the FilteredSet object specified by 'filter_id'.
"""
def clearFilteredSet(filter_id):
"""Clear the FilteredSet object specified by 'filter_id'.
"""
# IIndexConfiguration was added on request by the GenericSetup community in
# order to perform introspection on indexes in a defined way.
# (ajung)
class IIndexConfiguration(Interface):
""" Introspection API for pluggable indexes """
def getSettings(self):
""" Returns an mapping with index specific settings.
E.g. {'indexed_attrs' : ('SearchableText', )}.
The interface does not define any specifc mapping keys.
"""
src/Products/ZCatalog/Catalog.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import types
import logging
import warnings
from bisect import bisect
from random import randint
import Acquisition
from Acquisition import aq_base
from Acquisition import aq_parent
import ExtensionClass
from Missing import MV
from Persistence import Persistent
from Products.PluginIndexes.interfaces import ILimitedResultIndex
import BTrees.Length
from BTrees.IIBTree import intersection, weightedIntersection, IISet
from BTrees.OIBTree import OIBTree
from BTrees.IOBTree import IOBTree
from Lazy import LazyMap, LazyCat, LazyValues
from CatalogBrains import AbstractCatalogBrain, NoBrainer
from .plan import CatalogPlan
from .plan import make_key
LOG = logging.getLogger('Zope.ZCatalog')
try:
from DocumentTemplate.cDocumentTemplate import safe_callable
except ImportError:
# Fallback to python implementation to avoid dependancy on DocumentTemplate
def safe_callable(ob):
# Works with ExtensionClasses and Acquisition.
if hasattr(ob, '__class__'):
return hasattr(ob, '__call__') or isinstance(ob, types.ClassType)
else:
return callable(ob)
class CatalogError(Exception):
pass
class Catalog(Persistent, Acquisition.Implicit, ExtensionClass.Base):
""" An Object Catalog
An Object Catalog maintains a table of object metadata, and a
series of manageable indexes to quickly search for objects
(references in the metadata) that satisfy a search query.
This class is not Zope specific, and can be used in any python
program to build catalogs of objects. Note that it does require
the objects to be Persistent, and thus must be used with ZODB3.
"""
_v_brains = NoBrainer
def __init__(self, vocabulary=None, brains=None):
# Catalogs no longer care about vocabularies and lexicons
# so the vocabulary argument is ignored. (Casey)
self.schema = {} # mapping from attribute name to column number
self.names = () # sequence of column names
self.indexes = {} # maping from index name to index object
# The catalog maintains a BTree of object meta_data for
# convenient display on result pages. meta_data attributes
# are turned into brain objects and returned by
# searchResults. The indexing machinery indexes all records
# by an integer id (rid). self.data is a mapping from the
# integer id to the meta_data, self.uids is a mapping of the
# object unique identifier to the rid, and self.paths is a
# mapping of the rid to the unique identifier.
self.clear()
if brains is not None:
self._v_brains = brains
self.updateBrains()
def __len__(self):
return self._length()
def clear(self):
""" clear catalog """
self.data = IOBTree() # mapping of rid to meta_data
self.uids = OIBTree() # mapping of uid to rid
self.paths = IOBTree() # mapping of rid to uid
self._length = BTrees.Length.Length()
for index in self.indexes.keys():
self.getIndex(index).clear()
def updateBrains(self):
self.useBrains(self._v_brains)
def __getitem__(self, index, ttype=type(())):
"""
Returns instances of self._v_brains, or whatever is passed
into self.useBrains.
"""
if type(index) is ttype:
# then it contains a score...
normalized_score, score, key = index
r=self._v_result_class(self.data[key]).__of__(aq_parent(self))
r.data_record_id_ = key
r.data_record_score_ = score
r.data_record_normalized_score_ = normalized_score
else:
# otherwise no score, set all scores to 1
r=self._v_result_class(self.data[index]).__of__(aq_parent(self))
r.data_record_id_ = index
r.data_record_score_ = 1
r.data_record_normalized_score_ = 1
return r
def __setstate__(self, state):
""" initialize your brains. This method is called when the
catalog is first activated (from the persistent storage) """
Persistent.__setstate__(self, state)
self.updateBrains()
def useBrains(self, brains):
""" Sets up the Catalog to return an object (ala ZTables) that
is created on the fly from the tuple stored in the self.data
Btree.
"""
class mybrains(AbstractCatalogBrain, brains):
pass
scopy = self.schema.copy()
scopy['data_record_id_']=len(self.schema.keys())
scopy['data_record_score_']=len(self.schema.keys())+1
scopy['data_record_normalized_score_']=len(self.schema.keys())+2
mybrains.__record_schema__ = scopy
self._v_brains = brains
self._v_result_class = mybrains
def addColumn(self, name, default_value=None):
"""
adds a row to the meta data schema
"""
schema = self.schema
names = list(self.names)
if name in schema:
raise CatalogError('The column %s already exists' % name)
if name[0] == '_':
raise CatalogError('Cannot cache fields beginning with "_"')
values = schema.values()
if values:
schema[name] = max(values) + 1
else:
schema[name] = 0
names.append(name)
if default_value in (None, ''):
default_value = MV
for key, value in self.data.items():
rec = list(value)
rec.append(default_value)
self.data[key] = tuple(rec)
self.names = tuple(names)
self.schema = schema
# new column? update the brain
self.updateBrains()
self._p_changed = 1 # why?
def delColumn(self, name):
"""
deletes a row from the meta data schema
"""
names = list(self.names)
_index = names.index(name)
if not name in self.schema:
LOG.error('delColumn attempted to delete nonexistent '
'column %s.' % str(name))
return
del names[_index]
# rebuild the schema
i = 0
schema = {}
for name in names:
schema[name] = i
i = i + 1
self.schema = schema
self.names = tuple(names)
# update the brain
self.updateBrains()
# remove the column value from each record
for key, value in self.data.items():
rec = list(value)
del rec[_index]
self.data[key] = tuple(rec)
def addIndex(self, name, index_type):
"""Create a new index, given a name and a index_type.
Old format: index_type was a string, 'FieldIndex' 'TextIndex' or
'KeywordIndex' is no longer valid; the actual index must be
instantiated and passed in to addIndex.
New format: index_type is the actual index object to be stored.
"""
if name in self.indexes:
raise CatalogError('The index %s already exists' % name)
if name.startswith('_'):
raise CatalogError('Cannot index fields beginning with "_"')
if not name:
raise CatalogError('Name of index is empty')
indexes = self.indexes
if isinstance(index_type, str):
raise TypeError("Catalog addIndex now requires the index type to"
"be resolved prior to adding; create the proper "
"index in the caller.")
indexes[name] = index_type
self.indexes = indexes
def delIndex(self, name):
""" deletes an index """
if not name in self.indexes:
raise CatalogError('The index %s does not exist' % name)
indexes = self.indexes
del indexes[name]
self.indexes = indexes
def getIndex(self, name):
""" get an index wrapped in the catalog """
return self.indexes[name].__of__(self)
def updateMetadata(self, object, uid):
""" Given an object and a uid, update the column data for the
uid with the object data iff the object has changed """
data = self.data
index = self.uids.get(uid, None)
newDataRecord = self.recordify(object)
if index is None:
if type(data) is IOBTree:
# New style, get random id
index=getattr(self, '_v_nextid', 0)
if index % 4000 == 0:
index = randint(-2000000000, 2000000000)
while not data.insert(index, newDataRecord):
index = randint(-2000000000, 2000000000)
# We want ids to be somewhat random, but there are
# advantages for having some ids generated
# sequentially when many catalog updates are done at
# once, such as when reindexing or bulk indexing.
# We allocate ids sequentially using a volatile base,
# so different threads get different bases. This
# further reduces conflict and reduces churn in
# here and it result sets when bulk indexing.
self._v_nextid=index+1
else:
if data:
# find the next available unique id
index = data.keys()[-1] + 1
else:
index=0
# meta_data is stored as a tuple for efficiency
data[index] = newDataRecord
else:
if data.get(index, 0) != newDataRecord:
data[index] = newDataRecord
return index
# the cataloging API
def catalogObject(self, object, uid, threshold=None, idxs=None,
update_metadata=1):
"""
Adds an object to the Catalog by iteratively applying it to
all indexes.
'object' is the object to be cataloged
'uid' is the unique Catalog identifier for this object
If 'idxs' is specified (as a sequence), apply the object only
to the named indexes.
If 'update_metadata' is true (the default), also update metadata for
the object. If the object is new to the catalog, this flag has
no effect (metadata is always created for new objects).
"""
if idxs is None:
idxs = []
index = self.uids.get(uid, None)
if index is None: # we are inserting new data
index = self.updateMetadata(object, uid)
self._length.change(1)
self.uids[uid] = index
self.paths[index] = uid
elif update_metadata: # we are updating and we need to update metadata
self.updateMetadata(object, uid)
# do indexing
total = 0
if idxs == []:
use_indexes = self.indexes.keys()
else:
use_indexes = idxs
for name in use_indexes:
x = self.getIndex(name)
if hasattr(x, 'index_object'):
blah = x.index_object(index, object, threshold)
total = total + blah
else:
LOG.error('catalogObject was passed bad index '
'object %s.' % str(x))
return total
def uncatalogObject(self, uid):
"""
Uncatalog and object from the Catalog. and 'uid' is a unique
Catalog identifier
Note, the uid must be the same as when the object was
catalogued, otherwise it will not get removed from the catalog
This method should not raise an exception if the uid cannot
be found in the catalog.
"""
data = self.data
uids = self.uids
paths = self.paths
indexes = self.indexes.keys()
rid = uids.get(uid, None)
if rid is not None:
for name in indexes:
x = self.getIndex(name)
if hasattr(x, 'unindex_object'):
x.unindex_object(rid)
del data[rid]
del paths[rid]
del uids[uid]
self._length.change(-1)
else:
LOG.error('uncatalogObject unsuccessfully '
'attempted to uncatalog an object '
'with a uid of %s. ' % str(uid))
def uniqueValuesFor(self, name):
""" return unique values for FieldIndex name """
return self.getIndex(name).uniqueValues()
def hasuid(self, uid):
""" return the rid if catalog contains an object with uid """
return self.uids.get(uid)
def recordify(self, object):
""" turns an object into a record tuple """
record = []
# the unique id is always the first element
for x in self.names:
attr = getattr(object, x, MV)
if (attr is not MV and safe_callable(attr)):
attr = attr()
record.append(attr)
return tuple(record)
def instantiate(self, record):
r = self._v_result_class(record[1])
r.data_record_id_ = record[0]
return r.__of__(self)
def getMetadataForRID(self, rid):
record = self.data[rid]
result = {}
for (key, pos) in self.schema.items():
result[key] = record[pos]
return result
def getIndexDataForRID(self, rid):
result = {}
for name in self.indexes.keys():
result[name] = self.getIndex(name).getEntryForObject(rid, "")
return result
## This is the Catalog search engine. Most of the heavy lifting happens
# below
def make_query(self, request):
# This is a bit of a mess, but the ZCatalog API has traditionally
# supported passing in query restrictions in almost arbitary ways
real_req = None
if isinstance(request, dict):
query = request.copy()
elif isinstance(request, CatalogSearchArgumentsMap):
query = {}
query.update(request.keywords)
real_req = request.request
if isinstance(real_req, dict):
query.update(real_req)
real_req = None
else:
real_req = request
if real_req:
warnings.warn('You have specified a query using either a request '
'object or a mixture of a query dict and keyword '
'arguments. Please use only a simple query dict. '
'Your query contained "%s". This support is '
'deprecated and will be removed in Zope 2.14.' %
repr(real_req), DeprecationWarning, stacklevel=4)
known_keys = query.keys()
# The request has too many places where an index restriction
# might be specified. Putting all of request.form,
# request.other, ... into the query isn't what we want.
# So we iterate over all known indexes instead and see if they
# are in the request.
for iid in self.indexes.keys():
if iid in known_keys:
continue
value = real_req.get(iid)
if value:
query[iid] = value
return query
def _sorted_search_indexes(self, query):
# Simple implementation doing no ordering.
query_keys = query.keys()
order = []
for name, index in self.indexes.items():
if name not in query_keys:
continue
order.append((ILimitedResultIndex.providedBy(index), name))
order.sort()
return [i[1] for i in order]
def search(self, query, sort_index=None, reverse=0, limit=None, merge=1):
"""Iterate through the indexes, applying the query to each one. If
merge is true then return a lazy result set (sorted if appropriate)
otherwise return the raw (possibly scored) results for later merging.
Limit is used in conjuntion with sorting or scored results to inform
the catalog how many results you are really interested in. The catalog
can then use optimizations to save time and memory. The number of
results is not guaranteed to fall within the limit however, you should
still slice or batch the results as usual."""
rs = None # resultset
# Indexes fulfill a fairly large contract here. We hand each
# index the query mapping we are given (which may be composed
# of some combination of web request, kw mappings or plain old dicts)
# and the index decides what to do with it. If the index finds work
# for itself in the query, it returns the results and a tuple of
# the attributes that were used. If the index finds nothing for it
# to do then it returns None.
# Canonicalize the request into a sensible query before passing it on
query = self.make_query(query)
cr = self.getCatalogPlan(query)
cr.start()
plan = cr.plan()
if not plan:
plan = self._sorted_search_indexes(query)
indexes = self.indexes.keys()
for i in plan:
if i not in indexes:
# We can have bogus keys or the plan can contain index names
# that have been removed in the meantime
continue
index = self.getIndex(i)
_apply_index = getattr(index, "_apply_index", None)
if _apply_index is None:
continue
cr.start_split(i)
limit_result = ILimitedResultIndex.providedBy(index)
if limit_result:
r = _apply_index(query, rs)
else:
r = _apply_index(query)
if r is not None:
r, u = r
# Short circuit if empty result
# BBB: We can remove the "r is not None" check in Zope 2.14
# once we don't need to support the "return everything" case
# anymore
if r is not None and not r:
cr.stop_split(i, result=None, limit=limit_result)
return LazyCat([])
cr.stop_split(i, result=r, limit=limit_result)
w, rs = weightedIntersection(rs, r)
if not rs:
break
else:
cr.stop_split(i, result=None, limit=limit_result)
if rs is None:
# None of the indexes found anything to do with the query
# We take this to mean that the query was empty (an empty filter)
# and so we return everything in the catalog
warnings.warn('Your query %s produced no query restriction. '
'Currently the entire catalog content is returned. '
'In Zope 2.14 this will result in an empty LazyCat '
'to be returned.' % repr(make_key(self, query)),
DeprecationWarning, stacklevel=3)
if sort_index is None:
result = LazyMap(self.instantiate, self.data.items(), len(self))
else:
cr.start_split('sort_on')
result = self.sortResults(
self.data, sort_index, reverse, limit, merge)
cr.stop_split('sort_on', None)
elif rs:
# We got some results from the indexes.
# Sort and convert to sequences.
# XXX: The check for 'values' is really stupid since we call
# items() and *not* values()
if sort_index is None and hasattr(rs, 'values'):
# having a 'values' means we have a data structure with
# scores. Build a new result set, sort it by score, reverse
# it, compute the normalized score, and Lazify it.
if not merge:
# Don't bother to sort here, return a list of
# three tuples to be passed later to mergeResults
# note that data_record_normalized_score_ cannot be
# calculated and will always be 1 in this case
getitem = self.__getitem__
result = [(score, (1, score, rid), getitem)
for rid, score in rs.items()]
else:
cr.start_split('sort_on')
rs = rs.byValue(0) # sort it by score
max = float(rs[0][0])
# Here we define our getter function inline so that
# we can conveniently store the max value as a default arg
# and make the normalized score computation lazy
def getScoredResult(item, max=max, self=self):
"""
Returns instances of self._v_brains, or whatever is
passed into self.useBrains.
"""
score, key = item
r=self._v_result_class(self.data[key])\
.__of__(aq_parent(self))
r.data_record_id_ = key
r.data_record_score_ = score
r.data_record_normalized_score_ = int(100. * score / max)
return r
result = LazyMap(getScoredResult, rs, len(rs))
cr.stop_split('sort_on', None)
elif sort_index is None and not hasattr(rs, 'values'):
# no scores
if hasattr(rs, 'keys'):
rs = rs.keys()
result = LazyMap(self.__getitem__, rs, len(rs))
else:
# sort. If there are scores, then this block is not
# reached, therefore 'sort-on' does not happen in the
# context of a text index query. This should probably
# sort by relevance first, then the 'sort-on' attribute.
cr.start_split('sort_on')
result = self.sortResults(rs, sort_index, reverse, limit, merge)
cr.stop_split('sort_on', None)
else:
# Empty result set
result = LazyCat([])
cr.stop()
return result
def sortResults(self, rs, sort_index, reverse=0, limit=None, merge=1):
# Sort a result set using a sort index. Return a lazy
# result set in sorted order if merge is true otherwise
# returns a list of (sortkey, uid, getter_function) tuples
#
# The two 'for' loops in here contribute a significant
# proportion of the time to perform an indexed search.
# Try to avoid all non-local attribute lookup inside
# those loops.
assert limit is None or limit > 0, 'Limit value must be 1 or greater'
_intersection = intersection
_self__getitem__ = self.__getitem__
index_key_map = sort_index.documentToKeyMap()
_None = None
_keyerror = KeyError
result = []
append = result.append
if hasattr(rs, 'keys'):
rs = rs.keys()
rlen = len(rs)
if merge and limit is None and (
rlen > (len(sort_index) * (rlen / 100 + 1))):
# The result set is much larger than the sorted index,
# so iterate over the sorted index for speed.
# This is rarely exercised in practice...
length = 0
try:
intersection(rs, IISet(()))
except TypeError:
# rs is not an object in the IIBTree family.
# Try to turn rs into an IISet.
rs = IISet(rs)
for k, intset in sort_index.items():
# We have an index that has a set of values for
# each sort key, so we intersect with each set and
# get a sorted sequence of the intersections.
intset = _intersection(rs, intset)
if intset:
keys = getattr(intset, 'keys', _None)
if keys is not _None:
# Is this ever true?
intset = keys()
length += len(intset)
append((k, intset, _self__getitem__))
# Note that sort keys are unique.
if reverse:
result.sort(reverse=True)
else:
result.sort()
result = LazyCat(LazyValues(result), length)
elif limit is None or (limit * 4 > rlen):
# Iterate over the result set getting sort keys from the index
for did in rs:
try:
key = index_key_map[did]
except _keyerror:
# This document is not in the sort key index, skip it.
pass
else:
append((key, did, _self__getitem__))
# The reference back to __getitem__ is used in case
# we do not merge now and need to intermingle the
# results with those of other catalogs while avoiding
# the cost of instantiating a LazyMap per result
if merge:
if reverse:
result.sort(reverse=True)
else:
result.sort()
if limit is not None:
result = result[:limit]
result = LazyValues(result)
else:
return result
elif reverse:
# Limit/sort results using N-Best algorithm
# This is faster for large sets then a full sort
# And uses far less memory
keys = []
n = 0
worst = None
for did in rs:
try:
key = index_key_map[did]
except _keyerror:
# This document is not in the sort key index, skip it.
pass
else:
if n >= limit and key <= worst:
continue
i = bisect(keys, key)
keys.insert(i, key)
result.insert(i, (key, did, _self__getitem__))
if n == limit:
del keys[0], result[0]
else:
n += 1
worst = keys[0]
result.reverse()
if merge:
result = LazyValues(result)
else:
return result
elif not reverse:
# Limit/sort results using N-Best algorithm in reverse (N-Worst?)
keys = []
n = 0
best = None
for did in rs:
try:
key = index_key_map[did]
except _keyerror:
# This document is not in the sort key index, skip it.
pass
else:
if n >= limit and key >= best:
continue
i = bisect(keys, key)
keys.insert(i, key)
result.insert(i, (key, did, _self__getitem__))
if n == limit:
del keys[-1], result[-1]
else:
n += 1
best = keys[-1]
if merge:
result = LazyValues(result)
else:
return result
result = LazyMap(self.__getitem__, result, len(result))
result.actual_result_count = rlen
return result
def _get_sort_attr(self, attr, kw):
"""Helper function to find sort-on or sort-order."""
# There are three different ways to find the attribute:
# 1. kw[sort-attr]
# 2. self.sort-attr
# 3. kw[sort_attr]
# kw may be a dict or an ExtensionClass MultiMapping, which
# differ in what get() returns with no default value.
name = "sort-%s" % attr
val = kw.get(name, None)
if val is not None:
return val
val = getattr(self, name, None)
if val is not None:
return val
return kw.get("sort_%s" % attr, None)
def _getSortIndex(self, args):
"""Returns a search index object or None."""
sort_index_name = self._get_sort_attr("on", args)
if sort_index_name is not None:
# self.indexes is always a dict, so get() w/ 1 arg works
sort_index = self.indexes.get(sort_index_name)
if sort_index is None:
raise CatalogError('Unknown sort_on index (%s)' %
sort_index_name)
else:
if not hasattr(sort_index, 'documentToKeyMap'):
raise CatalogError(
'The index chosen for sort_on (%s) is not capable of '
'being used as a sort index.' % sort_index_name)
return sort_index
else:
return None
def searchResults(self, REQUEST=None, used=None, _merge=1, **kw):
# You should pass in a simple dictionary as the request argument,
# which only contains the relevant query.
# The used argument is deprecated and is ignored
if REQUEST is None and not kw:
# Try to acquire request if we get no args for bw compat
warnings.warn('Calling searchResults without a query argument nor '
'keyword arguments is deprecated. In Zope 2.14 the '
'query will no longer be automatically taken from '
'the acquired request.',
DeprecationWarning, stacklevel=3)
REQUEST = getattr(self, 'REQUEST', None)
if isinstance(REQUEST, dict) and not kw:
# short cut for the best practice
args = REQUEST
else:
args = CatalogSearchArgumentsMap(REQUEST, kw)
sort_index = self._getSortIndex(args)
sort_limit = self._get_sort_attr('limit', args)
reverse = 0
if sort_index is not None:
order = self._get_sort_attr("order", args)
if (isinstance(order, str) and
order.lower() in ('reverse', 'descending')):
reverse = 1
# Perform searches with indexes and sort_index
return self.search(args, sort_index, reverse, sort_limit, _merge)
__call__ = searchResults
def getCatalogPlan(self, query=None):
"""Query time reporting and planning.
"""
parent = aq_base(aq_parent(self))
threshold = getattr(parent, 'long_query_time', 0.1)
return CatalogPlan(self, query, threshold)
class CatalogSearchArgumentsMap(object):
"""Multimap catalog arguments coming simultaneously from keywords
and request.
BBB: Values that are empty strings are treated as non-existent. This is
to ignore empty values, thereby ignoring empty form fields to be
consistent with hysterical behavior. This is deprecated and can be changed
in Zope 2.14.
"""
def __init__(self, request, keywords):
self.request = request or {}
self.keywords = keywords or {}
def __getitem__(self, key):
marker = []
v = self.keywords.get(key, marker)
if v is marker or v == '':
v = self.request[key]
if v == '':
raise KeyError(key)
return v
def get(self, key, default=None):
try:
v = self[key]
except KeyError:
return default
else:
return v
def has_key(self, key):
try:
self[key]
except KeyError:
return False
else:
return True
def __contains__(self, name):
return self.has_key(name)
def mergeResults(results, has_sort_keys, reverse):
"""Sort/merge sub-results, generating a flat sequence.
results is a list of result set sequences, all with or without sort keys
"""
if not has_sort_keys:
return LazyCat(results)
else:
# Concatenate the catalog results into one list and sort it
# Each result record consists of a list of tuples with three values:
# (sortkey, docid, catalog__getitem__)
combined = []
if len(results) > 1:
for r in results:
combined.extend(r)
elif len(results) == 1:
combined = results[0]
else:
return []
if reverse:
combined.sort(reverse=True)
else:
combined.sort()
return LazyMap(lambda rec: rec[2](rec[1]), combined, len(combined))
src/Products/ZCatalog/CatalogAwareness.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""ZCatalog Findable class
**NOTE**: This module is deprecated, and should only be used for
backward-compatibility. All new code should use CatalogPathAwareness.
"""
import urllib
import warnings
from Acquisition import aq_base
from App.special_dtml import DTMLFile
class CatalogAware:
""" This is a Mix-In class to make objects automaticly catalog and
uncatalog themselves in Zope, and to provide some other basic
attributes that are useful to catalog. Note that if your class
subclasses CatalogAware, it will only catalog itself when
it is added or copied in Zope. If you make changes to your own
object, you are responsible for calling your object's index_object
method. """
meta_type='CatalogAware'
default_catalog='Catalog'
manage_editCatalogerForm=DTMLFile('dtml/editCatalogerForm', globals())
def _warn_deprecated(self):
warnings.warn('The Products.ZCatalog.CatalogAwareness module is '
'deprecated and will be removed in Zope 2.14. Please '
'use event subscribers for zope.lifecycle events to '
'automatically index and unindex your objects.',
DeprecationWarning, stacklevel=3)
def manage_editCataloger(self, default, REQUEST=None):
""" """
self.default_catalog=default
message = "Your changes have been saved"
if REQUEST is not None:
return self.manage_main(self, REQUEST, manage_tabs_message=message)
def manage_afterAdd(self, item, container):
self.index_object()
for object in self.objectValues():
try:
s = object._p_changed
except Exception:
s = 0
object.manage_afterAdd(item, container)
if s is None:
object._p_deactivate()
def manage_afterClone(self, item):
self.index_object()
for object in self.objectValues():
try:
s = object._p_changed
except Exception:
s = 0
object.manage_afterClone(item)
if s is None:
object._p_deactivate()
def manage_beforeDelete(self, item, container):
self.unindex_object()
for object in self.objectValues():
try:
s = object._p_changed
except Exception:
s = 0
object.manage_beforeDelete(item, container)
if s is None:
object._p_deactivate()
def creator(self):
"""Return a sequence of user names who have the local
Owner role on an object. The name creator is used
for this method to conform to Dublin Core."""
users=[]
for user, roles in self.get_local_roles():
if 'Owner' in roles:
users.append(user)
return ', '.join(users)
def onDeleteObject(self):
"""Object delete handler. I think this is obsoleted by
manage_beforeDelete """
self.unindex_object()
def url(self, ftype=urllib.splittype, fhost=urllib.splithost):
"""Return a SCRIPT_NAME-based url for an object."""
if hasattr(self, 'DestinationURL') and \
callable(self.DestinationURL):
url='%s/%s' % (self.DestinationURL(), self.id)
else:
url = self.absolute_url()
type, uri = ftype(url)
host, uri = fhost(uri)
script_name = self.REQUEST['SCRIPT_NAME']
if script_name:
uri = filter(None, uri.split(script_name))[0]
if not uri:
uri = '/'
if uri[0] != '/':
uri = '/' + uri
return urllib.unquote(uri)
def summary(self, num=200):
"""Return a summary of the text content of the object."""
if not hasattr(self, 'text_content'):
return ''
attr = getattr(self, 'text_content')
if callable(attr):
text = attr()
else:
text = attr
n = min(num, len(text))
return text[:n]
def index_object(self):
"""A common method to allow Findables to index themselves."""
self._warn_deprecated()
catalog = getattr(self, self.default_catalog, None)
if catalog is not None:
catalog.catalog_object(self, self.url())
def unindex_object(self):
"""A common method to allow Findables to unindex themselves."""
self._warn_deprecated()
catalog = getattr(self, self.default_catalog, None)
if catalog is not None:
catalog.uncatalog_object(self.url())
def reindex_object(self):
""" Suprisingly useful """
self.unindex_object()
self.index_object()
def reindex_all(self, obj=None):
""" """
if obj is None:
obj = self
if hasattr(aq_base(obj), 'index_object'):
obj.index_object()
if hasattr(aq_base(obj), 'objectValues'):
for item in obj.objectValues():
self.reindex_all(item)
return 'done!'
src/Products/ZCatalog/CatalogBrains.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
from zope.interface import implements
import Acquisition
from Acquisition import aq_parent
import Record
from interfaces import ICatalogBrain
class AbstractCatalogBrain(Record.Record, Acquisition.Implicit):
"""Abstract base brain that handles looking up attributes as
required, and provides just enough smarts to let us get the URL, path,
and cataloged object without having to ask the catalog directly.
"""
implements(ICatalogBrain)
def has_key(self, key):
return key in self.__record_schema__
def __contains__(self, name):
return name in self.__record_schema__
def getPath(self):
"""Get the physical path for this record"""
return aq_parent(self).getpath(self.data_record_id_)
def getURL(self, relative=0):
"""Generate a URL for this record"""
return self.REQUEST.physicalPathToURL(self.getPath(), relative)
def _unrestrictedGetObject(self):
"""Return the object for this record
Same as getObject, but does not do security checks.
"""
return aq_parent(self).unrestrictedTraverse(self.getPath())
def getObject(self, REQUEST=None):
"""Return the object for this record
Will return None if the object cannot be found via its cataloged path
(i.e., it was deleted or moved without recataloging), or if the user is
not authorized to access the object.
This method mimicks a subset of what publisher's traversal does,
so it allows access if the final object can be accessed even
if intermediate objects cannot.
"""
path = self.getPath().split('/')
if not path:
return None
parent = aq_parent(self)
if len(path) > 1:
parent = parent.unrestrictedTraverse(path[:-1])
return parent.restrictedTraverse(path[-1])
def getRID(self):
"""Return the record ID for this object."""
return self.data_record_id_
class NoBrainer:
""" This is an empty class to use when no brain is specified. """
pass
src/Products/ZCatalog/CatalogPathAwareness.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""ZCatalog Findable class
"""
import warnings
from Acquisition import aq_base
from App.special_dtml import DTMLFile
class CatalogAware:
""" This is a Mix-In class to make objects automaticly catalog and
uncatalog themselves in Zope, and to provide some other basic
attributes that are useful to catalog. Note that if your class
subclasses CatalogAware, it will only catalog itself when
it is added or copied in Zope. If you make changes to your own
object, you are responsible for calling your object's index_object
method. """
meta_type='CatalogAware'
default_catalog='Catalog'
manage_editCatalogerForm=DTMLFile('dtml/editCatalogerForm', globals())
def _warn_deprecated(self):
warnings.warn('The Products.ZCatalog.CatalogPathAwareness module is '
'deprecated and will be removed in Zope 2.14. Please '
'use event subscribers for zope.lifecycle events to '
'automatically index and unindex your objects.',
DeprecationWarning, stacklevel=3)
def manage_editCataloger(self, default, REQUEST=None):
""" """
self.default_catalog=default
message = "Your changes have been saved"
if REQUEST is not None:
return self.manage_main(self, REQUEST, manage_tabs_message=message)
def manage_afterAdd(self, item, container):
self.index_object()
for object in self.objectValues():
try:
s = object._p_changed
except Exception:
s = 0
object.manage_afterAdd(item, container)
if s is None:
object._p_deactivate()
def manage_afterClone(self, item):
self.index_object()
for object in self.objectValues():
try:
s = object._p_changed
except Exception:
s = 0
object.manage_afterClone(item)
if s is None:
object._p_deactivate()
def manage_beforeDelete(self, item, container):
self.unindex_object()
for object in self.objectValues():
try:
s = object._p_changed
except Exception:
s = 0
object.manage_beforeDelete(item, container)
if s is None:
object._p_deactivate()
def creator(self):
"""Return a sequence of user names who have the local
Owner role on an object. The name creator is used
for this method to conform to Dublin Core."""
users=[]
for user, roles in self.get_local_roles():
if 'Owner' in roles:
users.append(user)
return ', '.join(users)
def onDeleteObject(self):
"""Object delete handler. I think this is obsoleted by
manage_beforeDelete """
self.unindex_object()
def getPath(self):
"""Return the physical path for an object."""
return '/'.join(self.getPhysicalPath())
def summary(self, num=200):
"""Return a summary of the text content of the object."""
if not hasattr(self, 'text_content'):
return ''
attr=getattr(self, 'text_content')
if callable(attr):
text=attr()
else:
text=attr
n = min(num, len(text))
return text[:n]
def index_object(self):
"""A common method to allow Findables to index themselves."""
self._warn_deprecated()
if hasattr(self, self.default_catalog):
getattr(self,
self.default_catalog).catalog_object(self, self.getPath())
def unindex_object(self):
"""A common method to allow Findables to unindex themselves."""
self._warn_deprecated()
if hasattr(self, self.default_catalog):
getattr(self,
self.default_catalog).uncatalog_object(self.getPath())
def reindex_object(self):
""" Suprisingly useful """
self.unindex_object()
self.index_object()
def reindex_all(self, obj=None):
""" """
if obj is None:
obj = self
if hasattr(aq_base(obj), 'index_object'):
obj.index_object()
if hasattr(aq_base(obj), 'objectValues'):
for item in obj.objectValues():
self.reindex_all(item)
return 'done!'
src/Products/ZCatalog/Lazy.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
from itertools import islice, count
class Lazy(object):
# Allow (reluctantly) access to unprotected attributes
__allow_access_to_unprotected_subobjects__=1
def __repr__(self):
return repr(list(self))
def __len__(self):
# This is a worst-case len, subclasses should try to do better
try:
return self._len
except AttributeError:
pass
l = len(self._data)
while 1:
try:
self[l]
l = l + 1
except Exception:
self._len = l
return l
def __add__(self, other):
if not isinstance(other, Lazy):
raise TypeError(
"Can not concatenate objects. Both must be lazy sequences.")
return LazyCat([self, other])
def __getslice__(self, i1, i2):
r = []
for i in islice(count(i1), i2-i1):
try:
r.append(self[i])
except IndexError:
return r
return r
slice = __getslice__
class LazyCat(Lazy):
# Lazy concatenation of one or more sequences. Should be handy
# for accessing small parts of big searches.
def __init__(self, sequences, length=None):
if len(sequences) < 100:
# Optimize structure of LazyCats to avoid nesting
# We don't do this for large numbers of input sequences
# to make instantiation faster instead
flattened_seq = []
for s in sequences:
if isinstance(s, LazyCat):
# If one of the sequences passed is itself a LazyCat, add
# its base sequences rather than nest LazyCats
flattened_seq.extend(s._seq)
else:
flattened_seq.append(s)
sequences = flattened_seq
self._seq = sequences
self._data = []
self._sindex = 0
self._eindex = -1
if length is not None:
self._len = length
def __getitem__(self, index):
data = self._data
try:
seq = self._seq
except AttributeError:
return data[index]
i = index
if i < 0:
i = len(self) + i
if i < 0:
raise IndexError(index)
ind = len(data)
if i < ind:
return data[i]
ind = ind - 1
sindex = self._sindex
try:
s = seq[sindex]
except Exception:
raise IndexError(index)
eindex = self._eindex
while i > ind:
try:
eindex = eindex + 1
v = s[eindex]
data.append(v)
ind = ind + 1
except IndexError:
self._sindex = sindex = sindex + 1
try:
s = self._seq[sindex]
except Exception:
del self._seq
del self._sindex
del self._eindex
raise IndexError(index)
self._eindex = eindex = -1
self._eindex = eindex
return data[i]
def __len__(self):
# Make len of LazyCat only as expensive as the lens
# of its underlying sequences
try:
return self._len
except Exception:
try:
l = 0
for s in self._seq:
l += len(s)
except AttributeError:
l = len(self._data)
self._len = l
return l
class LazyMap(Lazy):
# Act like a sequence, but get data from a filtering process.
# Don't access data until necessary
def __init__(self, func, seq, length=None):
self._seq = seq
self._data = {}
self._func = func
if length is not None:
self._len = length
else:
self._len = len(seq)
def __getitem__(self, index):
data = self._data
if index in data:
return data[index]
value = data[index] = self._func(self._seq[index])
return value
class LazyFilter(Lazy):
# Act like a sequence, but get data from a filtering process.
# Don't access data until necessary. Only data for which test(data)
# returns true will be considered part of the set.
def __init__(self, test, seq):
self._seq = seq
self._data = []
self._eindex = -1
self._test = test
def __getitem__(self, index):
data = self._data
try:
s = self._seq
except AttributeError:
return data[index]
i = index
if i < 0:
i = len(self) + i
if i < 0:
raise IndexError(index)
ind = len(data)
if i < ind:
return data[i]
ind = ind - 1
test = self._test
e = self._eindex
while i > ind:
try:
e = e + 1
v = s[e]
if test(v):
data.append(v)
ind = ind + 1
except IndexError:
del self._test
del self._seq
del self._eindex
raise IndexError(index)
self._eindex = e
return data[i]
class LazyMop(Lazy):
# Act like a sequence, but get data from a filtering process.
# Don't access data until necessary. If the filter raises an exception
# for a given item, then that item isn't included in the sequence.
def __init__(self, test, seq):
self._seq = seq
self._data = []
self._eindex = -1
self._test = test
def __getitem__(self, index):
data = self._data
try:
s = self._seq
except AttributeError:
return data[index]
i = index
if i < 0:
i = len(self) + i
if i < 0:
raise IndexError(index)
ind = len(data)
if i < ind:
return data[i]
ind = ind - 1
test = self._test
e = self._eindex
while i > ind:
try:
e = e + 1
v = s[e]
try:
v = test(v)
data.append(v)
ind = ind + 1
except Exception:
pass
except IndexError:
del self._test
del self._seq
del self._eindex
raise IndexError(index)
self._eindex = e
return data[i]
class LazyValues(Lazy):
"""Given a sequence of two tuples typically (key, value) act as
though we are just a list of the values lazily"""
def __init__(self, seq):
self._seq = seq
def __len__(self):
return len(self._seq)
def __getitem__(self, index):
return self._seq[index][1]
def __getslice__(self, start, end):
return self.__class__(self._seq[start:end])
slice = __getslice__
src/Products/ZCatalog/ProgressHandler.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import sys
import time
from logging import getLogger
from DateTime.DateTime import DateTime
from zope.interface import implements
from .interfaces import IProgressHandler
LOG = getLogger('ProgressHandler')
class StdoutHandler(object):
""" A simple progress handler """
implements(IProgressHandler)
def __init__(self, steps=100):
self._steps = steps
def init(self, ident, max):
self._ident = ident
self._max = max
self._start = time.time()
self.fp = sys.stdout
self.output('Process started (%d objects to go)' % self._max)
def info(self, text):
self.output(text)
def finish(self):
self.output('Process terminated. Duration: %0.2f seconds' % \
(time.time() -self._start))
def report(self, current, *args, **kw):
if current > 0:
if current % self._steps == 0:
seconds_so_far = time.time() - self._start
seconds_to_go = (seconds_so_far / current *
(self._max - current))
end = DateTime(time.time() + seconds_to_go)
self.output('%d/%d (%.2f%%) Estimated termination: %s' % \
(current, self._max, (100.0 * current / self._max),
end.strftime('%Y/%m/%d %H:%M:%Sh')))
def output(self, text):
print >>self.fp, '%s: %s' % (self._ident, text)
class ZLogHandler(StdoutHandler):
""" Use Zope logger"""
def output(self, text):
LOG.info(text)
class FilelogHandler(StdoutHandler):
""" Use a custom file for logging """
def __init__(self, filename, steps=100):
StdoutHandler.__init__(self, steps)
self.filename = filename
def output(self, text):
open(self.filename, 'a').write(text + '\n')
src/Products/ZCatalog/ZCatalog.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
""" ZCatalog product
"""
import logging
import operator
import sys
import string
import time
import urllib
from AccessControl.class_init import InitializeClass
from AccessControl.Permission import name_trans
from AccessControl.Permissions import manage_zcatalog_entries
from AccessControl.Permissions import manage_zcatalog_indexes
from AccessControl.Permissions import search_zcatalog
from AccessControl.SecurityInfo import ClassSecurityInfo
from Acquisition import aq_base
from Acquisition import aq_parent
from Acquisition import Implicit
from App.Dialogs import MessageDialog
from App.special_dtml import DTMLFile
from DateTime.DateTime import DateTime
from DocumentTemplate.DT_Util import InstanceDict
from DocumentTemplate.DT_Util import TemplateDict
from DocumentTemplate.DT_Util import Eval
from DocumentTemplate.security import RestrictedDTML
from OFS.Folder import Folder
from OFS.ObjectManager import ObjectManager
from Persistence import Persistent
from Products.PluginIndexes.interfaces import IPluggableIndex
import transaction
from ZODB.POSException import ConflictError
from zope.interface import implements
from Products.ZCatalog.Catalog import Catalog, CatalogError
from Products.ZCatalog.interfaces import IZCatalog
from Products.ZCatalog.ProgressHandler import ZLogHandler
from Products.ZCatalog.ZCatalogIndexes import ZCatalogIndexes
from .plan import PriorityMap
LOG = logging.getLogger('Zope.ZCatalog')
manage_addZCatalogForm = DTMLFile('dtml/addZCatalog', globals())
def manage_addZCatalog(self, id, title, vocab_id=None, REQUEST=None):
"""Add a ZCatalog object. The vocab_id argument is ignored.
"""
id = str(id)
title = str(title)
c = ZCatalog(id, title, container=self)
self._setObject(id, c)
if REQUEST is not None:
return self.manage_main(self, REQUEST, update_menu=1)
class ZCatalog(Folder, Persistent, Implicit):
"""ZCatalog object
A ZCatalog contains arbirary index like references to Zope
objects. ZCatalog's can index either 'Field' values of object, or
'Text' values.
ZCatalog does not store references to the objects themselves, but
rather to a unique identifier that defines how to get to the
object. In Zope, this unique idenfier is the object's relative
path to the ZCatalog (since two Zope object's cannot have the same
URL, this is an excellent unique qualifier in Zope).
Most of the dirty work is done in the _catalog object, which is an
instance of the Catalog class. An interesting feature of this
class is that it is not Zope specific. You can use it in any
Python program to catalog objects.
"""
implements(IZCatalog)
security = ClassSecurityInfo()
security.setPermissionDefault(manage_zcatalog_entries, ('Manager', ))
security.setPermissionDefault(manage_zcatalog_indexes, ('Manager', ))
security.setPermissionDefault(search_zcatalog, ('Anonymous', 'Manager'))
security.declareProtected(search_zcatalog, 'all_meta_types')
meta_type = "ZCatalog"
icon = 'misc_/ZCatalog/ZCatalog.gif'
manage_options = (
{'label': 'Contents', 'action': 'manage_main'},
{'label': 'Catalog', 'action': 'manage_catalogView'},
{'label': 'Properties', 'action': 'manage_propertiesForm'},
{'label': 'Indexes', 'action': 'manage_catalogIndexes'},
{'label': 'Metadata', 'action': 'manage_catalogSchema'},
{'label': 'Find Objects', 'action': 'manage_catalogFind'},
{'label': 'Advanced', 'action': 'manage_catalogAdvanced'},
{'label': 'Query Report', 'action': 'manage_catalogReport'},
{'label': 'Query Plan', 'action': 'manage_catalogPlan'},
{'label': 'Undo', 'action': 'manage_UndoForm'},
{'label': 'Security', 'action': 'manage_access'},
{'label': 'Ownership', 'action': 'manage_owner'},
)
security.declareProtected(manage_zcatalog_entries, 'manage_main')
security.declareProtected(manage_zcatalog_entries, 'manage_catalogView')
manage_catalogView = DTMLFile('dtml/catalogView', globals())
security.declareProtected(manage_zcatalog_entries, 'manage_catalogIndexes')
manage_catalogIndexes = DTMLFile('dtml/catalogIndexes', globals())
security.declareProtected(manage_zcatalog_entries, 'manage_catalogSchema')
manage_catalogSchema = DTMLFile('dtml/catalogSchema', globals())
security.declareProtected(manage_zcatalog_entries, 'manage_catalogFind')
manage_catalogFind = DTMLFile('dtml/catalogFind', globals())
security.declareProtected(manage_zcatalog_entries,
'manage_catalogAdvanced')
manage_catalogAdvanced = DTMLFile('dtml/catalogAdvanced', globals())
security.declareProtected(manage_zcatalog_entries, 'manage_catalogReport')
manage_catalogReport = DTMLFile('dtml/catalogReport', globals())
security.declareProtected(manage_zcatalog_entries, 'manage_catalogPlan')
manage_catalogPlan = DTMLFile('dtml/catalogPlan', globals())
security.declareProtected(manage_zcatalog_entries,
'manage_objectInformation')
manage_objectInformation = DTMLFile('dtml/catalogObjectInformation',
globals())
Indexes = ZCatalogIndexes()
threshold = 10000
long_query_time = 0.1
# vocabulary and vocab_id are left for backwards
# compatibility only, they are not used anymore
vocabulary = None
vocab_id = ''
_v_total = 0
_v_transaction = None
def __init__(self, id, title='', vocab_id=None, container=None):
# ZCatalog no longer cares about vocabularies
# so the vocab_id argument is ignored (Casey)
if container is not None:
self = self.__of__(container)
self.id=id
self.title=title
self.threshold = 10000
self.long_query_time = 0.1 # in seconds
self._v_total = 0
self._catalog = Catalog()
def __len__(self):
return len(self._catalog)
def manage_edit(self, RESPONSE, URL1, threshold=1000, REQUEST=None):
""" edit the catalog """
if not isinstance(threshold, int):
threshold = int(threshold)
self.threshold = threshold
RESPONSE.redirect(
URL1 + '/manage_main?manage_tabs_message=Catalog%20Changed')
def manage_subbingToggle(self, REQUEST, RESPONSE, URL1):
""" toggle subtransactions """
if self.threshold:
self.threshold = None
else:
self.threshold = 10000
RESPONSE.redirect(
URL1 +
'/manage_catalogAdvanced?manage_tabs_message=Catalog%20Changed')
security.declareProtected(manage_zcatalog_entries, 'manage_catalogObject')
def manage_catalogObject(self, REQUEST, RESPONSE, URL1, urls=None):
""" index Zope object(s) that 'urls' point to """
if urls:
if isinstance(urls, str):
urls = (urls, )
for url in urls:
obj = self.resolve_path(url)
if obj is None and hasattr(self, 'REQUEST'):
obj = self.resolve_url(url, REQUEST)
if obj is not None:
self.catalog_object(obj, url)
RESPONSE.redirect(
URL1 +
'/manage_catalogView?manage_tabs_message=Object%20Cataloged')
security.declareProtected(manage_zcatalog_entries,
'manage_uncatalogObject')
def manage_uncatalogObject(self, REQUEST, RESPONSE, URL1, urls=None):
""" removes Zope object(s) 'urls' from catalog """
if urls:
if isinstance(urls, str):
urls = (urls, )
for url in urls:
self.uncatalog_object(url)
RESPONSE.redirect(
URL1 +
'/manage_catalogView?manage_tabs_message=Object%20Uncataloged')
security.declareProtected(manage_zcatalog_entries, 'manage_catalogReindex')
def manage_catalogReindex(self, REQUEST, RESPONSE, URL1):
""" clear the catalog, then re-index everything """
elapse = time.time()
c_elapse = time.clock()
pgthreshold = self._getProgressThreshold()
handler = (pgthreshold > 0) and ZLogHandler(pgthreshold) or None
self.refreshCatalog(clear=1, pghandler=handler)
elapse = time.time() - elapse
c_elapse = time.clock() - c_elapse
RESPONSE.redirect(
URL1 +
'/manage_catalogAdvanced?manage_tabs_message=' +
urllib.quote('Catalog Updated \n'
'Total time: %s\n'
'Total CPU time: %s' % (`elapse`, `c_elapse`)))
security.declareProtected(manage_zcatalog_entries, 'refreshCatalog')
def refreshCatalog(self, clear=0, pghandler=None):
""" re-index everything we can find """
cat = self._catalog
paths = cat.paths.values()
if clear:
paths = tuple(paths)
cat.clear()
num_objects = len(paths)
if pghandler:
pghandler.init('Refreshing catalog: %s' % self.absolute_url(1),
num_objects)
for i in xrange(num_objects):
if pghandler:
pghandler.report(i)
p = paths[i]
obj = self.resolve_path(p)
if obj is None:
obj = self.resolve_url(p, self.REQUEST)
if obj is not None:
try:
self.catalog_object(obj, p, pghandler=pghandler)
except ConflictError:
raise
except Exception:
LOG.error('Recataloging object at %s failed' % p,
exc_info=sys.exc_info())
if pghandler:
pghandler.finish()
security.declareProtected(manage_zcatalog_entries, 'manage_catalogClear')
def manage_catalogClear(self, REQUEST=None, RESPONSE=None, URL1=None):
""" clears the whole enchilada """
self._catalog.clear()
if REQUEST and RESPONSE:
RESPONSE.redirect(
URL1 +
'/manage_catalogAdvanced?manage_tabs_message=Catalog%20Cleared')
security.declareProtected(manage_zcatalog_entries,
'manage_catalogFoundItems')
def manage_catalogFoundItems(self, REQUEST, RESPONSE, URL2, URL1,
obj_metatypes=None,
obj_ids=None, obj_searchterm=None,
obj_expr=None, obj_mtime=None,
obj_mspec=None, obj_roles=None,
obj_permission=None):
""" Find object according to search criteria and Catalog them
"""
elapse = time.time()
c_elapse = time.clock()
obj = REQUEST.PARENTS[1]
path = '/'.join(obj.getPhysicalPath())
self.ZopeFindAndApply(obj,
obj_metatypes=obj_metatypes,
obj_ids=obj_ids,
obj_searchterm=obj_searchterm,
obj_expr=obj_expr,
obj_mtime=obj_mtime,
obj_mspec=obj_mspec,
obj_permission=obj_permission,
obj_roles=obj_roles,
search_sub=1,
REQUEST=REQUEST,
apply_func=self.catalog_object,
apply_path=path)
elapse = time.time() - elapse
c_elapse = time.clock() - c_elapse
RESPONSE.redirect(
URL1 +
'/manage_catalogView?manage_tabs_message=' +
urllib.quote('Catalog Updated\n'
'Total time: %s\n'
'Total CPU time: %s'
% (`elapse`, `c_elapse`)))
security.declareProtected(manage_zcatalog_entries, 'manage_addColumn')
def manage_addColumn(self, name, REQUEST=None, RESPONSE=None, URL1=None):
""" add a column """
self.addColumn(name)
if REQUEST and RESPONSE:
RESPONSE.redirect(
URL1 +
'/manage_catalogSchema?manage_tabs_message=Column%20Added')
security.declareProtected(manage_zcatalog_entries, 'manage_delColumn')
def manage_delColumn(self, names, REQUEST=None, RESPONSE=None, URL1=None):
""" delete a column or some columns """
if isinstance(names, str):
names = (names, )
for name in names:
self.delColumn(name)
if REQUEST and RESPONSE:
RESPONSE.redirect(
URL1 +
'/manage_catalogSchema?manage_tabs_message=Column%20Deleted')
security.declareProtected(manage_zcatalog_entries, 'manage_addIndex')
def manage_addIndex(self, name, type, extra=None,
REQUEST=None, RESPONSE=None, URL1=None):
"""add an index """
self.addIndex(name, type, extra)
if REQUEST and RESPONSE:
RESPONSE.redirect(
URL1 +
'/manage_catalogIndexes?manage_tabs_message=Index%20Added')
security.declareProtected(manage_zcatalog_entries, 'manage_delIndex')
def manage_delIndex(self, ids=None, REQUEST=None, RESPONSE=None,
URL1=None):
""" delete an index or some indexes """
if not ids:
return MessageDialog(title='No items specified',
message='No items were specified!',
action="./manage_catalogIndexes")
if isinstance(ids, str):
ids = (ids, )
for name in ids:
self.delIndex(name)
if REQUEST and RESPONSE:
RESPONSE.redirect(
URL1 +
'/manage_catalogIndexes?manage_tabs_message=Index%20Deleted')
security.declareProtected(manage_zcatalog_entries, 'manage_clearIndex')
def manage_clearIndex(self, ids=None, REQUEST=None, RESPONSE=None,
URL1=None):
""" clear an index or some indexes """
if not ids:
return MessageDialog(title='No items specified',
message='No items were specified!',
action="./manage_catalogIndexes")
if isinstance(ids, str):
ids = (ids, )
for name in ids:
self.clearIndex(name)
if REQUEST and RESPONSE:
RESPONSE.redirect(
URL1 +
'/manage_catalogIndexes?manage_tabs_message=Index%20Cleared')
def reindexIndex(self, name, REQUEST, pghandler=None):
if isinstance(name, str):
name = (name, )
paths = self._catalog.uids.keys()
i = 0
if pghandler:
pghandler.init('reindexing %s' % name, len(paths))
for p in paths:
i += 1
if pghandler:
pghandler.report(i)
obj = self.resolve_path(p)
if obj is None:
obj = self.resolve_url(p, REQUEST)
if obj is None:
LOG.error('reindexIndex could not resolve '
'an object from the uid %r.' % p)
else:
# don't update metadata when only reindexing a single
# index via the UI
self.catalog_object(obj, p, idxs=name,
update_metadata=0, pghandler=pghandler)
if pghandler:
pghandler.finish()
security.declareProtected(manage_zcatalog_entries, 'manage_reindexIndex')
def manage_reindexIndex(self, ids=None, REQUEST=None, RESPONSE=None,
URL1=None):
"""Reindex indexe(s) from a ZCatalog"""
if not ids:
return MessageDialog(title='No items specified',
message='No items were specified!',
action="./manage_catalogIndexes")
pgthreshold = self._getProgressThreshold()
handler = (pgthreshold > 0) and ZLogHandler(pgthreshold) or None
self.reindexIndex(ids, REQUEST, handler)
if REQUEST and RESPONSE:
RESPONSE.redirect(
URL1 +
'/manage_catalogIndexes'
'?manage_tabs_message=Reindexing%20Performed')
security.declareProtected(manage_zcatalog_entries, 'catalog_object')
def catalog_object(self, obj, uid=None, idxs=None, update_metadata=1,
pghandler=None):
""" wrapper around catalog """
if uid is None:
try:
uid = obj.getPhysicalPath
except AttributeError:
raise CatalogError(
"A cataloged object must support the 'getPhysicalPath' "
"method if no unique id is provided when cataloging")
else:
uid = '/'.join(uid())
elif not isinstance(uid, str):
raise CatalogError('The object unique id must be a string.')
self._catalog.catalogObject(obj, uid, None, idxs,
update_metadata=update_metadata)
# None passed in to catalogObject as third argument indicates
# that we shouldn't try to commit subtransactions within any
# indexing code. We throw away the result of the call to
# catalogObject (which is a word count), because it's
# worthless to us here.
if self.threshold is not None:
# figure out whether or not to commit a subtransaction.
t = id(transaction.get())
if t != self._v_transaction:
self._v_total = 0
self._v_transaction = t
self._v_total = self._v_total + 1
# increment the _v_total counter for this thread only and get
# a reference to the current transaction.
# the _v_total counter is zeroed if we notice that we're in
# a different transaction than the last one that came by.
# self.threshold represents the number of times that
# catalog_object needs to be called in order for the catalog
# to commit a subtransaction. The semantics here mean that
# we should commit a subtransaction if our threshhold is
# exceeded within the boundaries of the current transaction.
if self._v_total > self.threshold:
transaction.savepoint(optimistic=True)
self._p_jar.cacheGC()
self._v_total = 0
if pghandler:
pghandler.info('committing subtransaction')
security.declareProtected(manage_zcatalog_entries, 'uncatalog_object')
def uncatalog_object(self, uid):
"""Wrapper around catalog """
self._catalog.uncatalogObject(uid)
security.declareProtected(search_zcatalog, 'uniqueValuesFor')
def uniqueValuesFor(self, name):
"""Return the unique values for a given FieldIndex """
return self._catalog.uniqueValuesFor(name)
security.declareProtected(search_zcatalog, 'getpath')
def getpath(self, rid):
"""Return the path to a cataloged object given a 'data_record_id_'
"""
return self._catalog.paths[rid]
def getrid(self, path, default=None):
"""Return 'data_record_id_' the to a cataloged object given a 'path'
"""
return self._catalog.uids.get(path, default)
security.declareProtected(search_zcatalog, 'getobject')
def getobject(self, rid, REQUEST=None):
"""Return a cataloged object given a 'data_record_id_'
"""
return aq_parent(self).unrestrictedTraverse(self.getpath(rid))
def getMetadataForUID(self, uid):
"""return the correct metadata given the uid, usually the path"""
rid = self._catalog.uids[uid]
return self._catalog.getMetadataForRID(rid)
def getIndexDataForUID(self, uid):
"""return the current index contents given the uid, usually the path"""
rid = self._catalog.uids[uid]
return self._catalog.getIndexDataForRID(rid)
def getMetadataForRID(self, rid):
"""return the correct metadata for the cataloged record id"""
return self._catalog.getMetadataForRID(int(rid))
def getIndexDataForRID(self, rid):
"""return the current index contents for the specific rid"""
return self._catalog.getIndexDataForRID(rid)
security.declareProtected(search_zcatalog, 'schema')
def schema(self):
return self._catalog.schema.keys()
security.declareProtected(search_zcatalog, 'indexes')
def indexes(self):
return self._catalog.indexes.keys()
security.declareProtected(search_zcatalog, 'index_objects')
def index_objects(self):
# This method returns unwrapped indexes!
# You should probably use getIndexObjects instead
return self._catalog.indexes.values()
security.declareProtected(manage_zcatalog_indexes, 'getIndexObjects')
def getIndexObjects(self):
# Return a list of wrapped(!) indexes
getIndex = self._catalog.getIndex
return [getIndex(name) for name in self.indexes()]
def _searchable_arguments(self):
r = {}
n = {'optional': 1}
for name in self._catalog.indexes.keys():
r[name] = n
return r
def _searchable_result_columns(self):
r = []
for name in self._catalog.schema.keys():
i = {}
i['name'] = name
i['type'] = 's'
i['parser'] = str
i['width'] = 8
r.append(i)
r.append({'name': 'data_record_id_',
'type': 's',
'parser': str,
'width': 8})
return r
security.declareProtected(search_zcatalog, 'searchResults')
def searchResults(self, REQUEST=None, used=None, **kw):
"""Search the catalog
Search terms can be passed in the REQUEST or as keyword
arguments.
The used argument is now deprecated and ignored
"""
return self._catalog.searchResults(REQUEST, used, **kw)
security.declareProtected(search_zcatalog, '__call__')
__call__ = searchResults
security.declareProtected(search_zcatalog, 'search')
def search(
self, query_request, sort_index=None, reverse=0, limit=None, merge=1):
"""Programmatic search interface, use for searching the catalog from
scripts.
query_request: Dictionary containing catalog query
sort_index: Name of sort index
reverse: Reverse sort order?
limit: Limit sorted result count (optimization hint)
merge: Return merged results (like searchResults) or raw
results for later merging.
"""
if sort_index is not None:
sort_index = self._catalog.indexes[sort_index]
return self._catalog.search(
query_request, sort_index, reverse, limit, merge)
## this stuff is so the find machinery works
meta_types=() # Sub-object types that are specific to this object
security.declareProtected(search_zcatalog, 'valid_roles')
def valid_roles(self):
"Return list of valid roles"
obj=self
dict={}
dup =dict.has_key
x=0
while x < 100:
if hasattr(obj, '__ac_roles__'):
roles=obj.__ac_roles__
for role in roles:
if not dup(role):
dict[role]=1
obj = aq_parent(obj)
if obj is None:
break
x = x + 1
roles=dict.keys()
roles.sort()
return roles
def ZopeFindAndApply(self, obj, obj_ids=None, obj_metatypes=None,
obj_searchterm=None, obj_expr=None,
obj_mtime=None, obj_mspec=None,
obj_permission=None, obj_roles=None,
search_sub=0,
REQUEST=None, result=None, pre='',
apply_func=None, apply_path=''):
"""Zope Find interface and apply
This is a *great* hack. Zope find just doesn't do what we
need here; the ability to apply a method to all the objects
*as they're found* and the need to pass the object's path into
that method.
"""
if result is None:
result = []
if obj_metatypes and 'all' in obj_metatypes:
obj_metatypes = None
if obj_mtime and isinstance(obj_mtime, str):
obj_mtime = DateTime(obj_mtime).timeTime()
if obj_permission:
obj_permission = p_name(obj_permission)
if obj_roles and isinstance(obj_roles, str):
obj_roles = [obj_roles]
if obj_expr:
# Setup expr machinations
md = td()
obj_expr = (Eval(obj_expr), md, md._push, md._pop)
base = aq_base(obj)
if not hasattr(base, 'objectItems'):
return result
try:
items = obj.objectItems()
except Exception:
return result
try:
add_result = result.append
except Exception:
raise AttributeError(repr(result))
for id, ob in items:
if pre:
p = "%s/%s" % (pre, id)
else:
p = id
dflag = 0
if hasattr(ob, '_p_changed') and (ob._p_changed == None):
dflag = 1
bs = aq_base(ob)
if (
(not obj_ids or absattr(bs.id) in obj_ids)
and
(not obj_metatypes or (hasattr(bs, 'meta_type') and
bs.meta_type in obj_metatypes))
and
(not obj_searchterm or
(hasattr(ob, 'PrincipiaSearchSource') and
ob.PrincipiaSearchSource().find(obj_searchterm) >= 0))
and
(not obj_expr or expr_match(ob, obj_expr))
and
(not obj_mtime or mtime_match(ob, obj_mtime, obj_mspec))
and
((not obj_permission or not obj_roles) or
role_match(ob, obj_permission, obj_roles))
):
if apply_func:
apply_func(ob, (apply_path + '/' + p))
else:
add_result((p, ob))
dflag = 0
if search_sub and hasattr(bs, 'objectItems'):
self.ZopeFindAndApply(ob, obj_ids, obj_metatypes,
obj_searchterm, obj_expr,
obj_mtime, obj_mspec,
obj_permission, obj_roles,
search_sub,
REQUEST, result, p,
apply_func, apply_path)
if dflag:
ob._p_deactivate()
return result
security.declareProtected(search_zcatalog, 'resolve_url')
def resolve_url(self, path, REQUEST):
"""
Attempt to resolve a url into an object in the Zope
namespace. The url may be absolute or a catalog path
style url. If no object is found, None is returned.
No exceptions are raised.
"""
if REQUEST:
script=REQUEST.script
if path.find(script) != 0:
path='%s/%s' % (script, path)
try:
return REQUEST.resolve_url(path)
except Exception:
pass
def resolve_path(self, path):
"""
Attempt to resolve a url into an object in the Zope
namespace. The url may be absolute or a catalog path
style url. If no object is found, None is returned.
No exceptions are raised.
"""
try:
return self.unrestrictedTraverse(path)
except Exception:
pass
def manage_normalize_paths(self, REQUEST):
"""Ensure that all catalog paths are full physical paths
This should only be used with ZCatalogs in which all paths can
be resolved with unrestrictedTraverse."""
paths = self._catalog.paths
uids = self._catalog.uids
unchanged = 0
fixed = []
removed = []
for path, rid in uids.items():
ob = None
if path[:1] == '/':
ob = self.resolve_url(path[1:], REQUEST)
if ob is None:
ob = self.resolve_url(path, REQUEST)
if ob is None:
removed.append(path)
continue
ppath = '/'.join(ob.getPhysicalPath())
if path != ppath:
fixed.append((path, ppath))
else:
unchanged = unchanged + 1
for path, ppath in fixed:
rid = uids[path]
del uids[path]
paths[rid] = ppath
uids[ppath] = rid
for path in removed:
self.uncatalog_object(path)
return MessageDialog(title='Done Normalizing Paths',
message='%s paths normalized, %s paths removed, and '
'%s unchanged.' % (len(fixed), len(removed), unchanged),
action='./manage_main')
security.declareProtected(manage_zcatalog_entries, 'manage_setProgress')
def manage_setProgress(self, pgthreshold=0, RESPONSE=None, URL1=None):
"""Set parameter to perform logging of reindexing operations very
'pgthreshold' objects
"""
self.pgthreshold = pgthreshold
if RESPONSE:
RESPONSE.redirect(URL1 + '/manage_catalogAdvanced?'
'manage_tabs_message=Catalog%20Changed')
def _getProgressThreshold(self):
if not hasattr(self, 'pgthreshold'):
self.pgthreshold = 0
return self.pgthreshold
# Indexing methods
def addIndex(self, name, type, extra=None):
# Convert the type by finding an appropriate product which supports
# this interface by that name. Bleah
products = ObjectManager.all_meta_types(self,
interfaces=(IPluggableIndex, ))
p = None
for prod in products:
if prod['name'] == type:
p = prod
break
if p is None:
raise ValueError("Index of type %s not found" % type)
base = p['instance']
if base is None:
raise ValueError("Index type %s does not support addIndex" % type)
# This code is *really* lame but every index type has its own
# function signature *sigh* and there is no common way to pass
# additional parameters to the constructor. The suggested way
# for new index types is to use an "extra" record.
if 'extra' in base.__init__.func_code.co_varnames:
index = base(name, extra=extra, caller=self)
elif 'caller' in base.__init__.func_code.co_varnames:
index = base(name, caller=self)
else:
index = base(name)
self._catalog.addIndex(name, index)
def delIndex(self, name):
self._catalog.delIndex(name)
def clearIndex(self, name):
self._catalog.getIndex(name).clear()
def addColumn(self, name, default_value=None):
return self._catalog.addColumn(name, default_value)
def delColumn(self, name):
return self._catalog.delColumn(name)
# Catalog plan methods
security.declareProtected(manage_zcatalog_entries, 'getCatalogPlan')
def getCatalogPlan(self):
"""Get a string representation of a query plan"""
pmap = PriorityMap.get_value()
output = []
output.append('# query plan dumped at %r\n' % time.asctime())
output.append('queryplan = {')
for cid, plan in sorted(pmap.items()):
output.append(' %s: {' % repr(cid))
for querykey, details in sorted(plan.items()):
output.append(' %s: {' % repr(querykey))
for indexname, benchmark in sorted(details.items()):
tuplebench = repr(tuple(benchmark))
output.append(' %r:\n %s,' % (indexname, tuplebench))
output.append(' },')
output.append(' },')
output.append('}')
return '\n'.join(output)
security.declareProtected(manage_zcatalog_entries, 'getCatalogReport')
def getCatalogReport(self):
"""Query time reporting."""
rval = self._catalog.getCatalogPlan().report()
rval.sort(key=operator.itemgetter('duration'), reverse=True)
return rval
security.declareProtected(manage_zcatalog_entries,
'manage_resetCatalogReport')
def manage_resetCatalogReport(self, REQUEST=None):
"""Resets the catalog report."""
self._catalog.getCatalogPlan().reset()
if REQUEST is not None:
REQUEST.response.redirect(REQUEST.URL1 +
'/manage_catalogReport?manage_tabs_message=Report%20cleared')
security.declareProtected(manage_zcatalog_entries,
'manage_editCatalogReport')
def manage_editCatalogReport(self, long_query_time=0.1, REQUEST=None):
"""Edit the long query time."""
if not isinstance(long_query_time, float):
long_query_time = float(long_query_time)
self.long_query_time = long_query_time
if REQUEST is not None:
REQUEST.response.redirect(REQUEST.URL1 +
'/manage_catalogReport?manage_tabs_message=' +
'Long%20query%20time%20changed')
InitializeClass(ZCatalog)
def p_name(name):
return '_' + string.translate(name, name_trans) + '_Permission'
def absattr(attr):
if callable(attr):
return attr()
return attr
class td(RestrictedDTML, TemplateDict):
pass
def expr_match(ob, ed):
e, md, push, pop = ed
push(InstanceDict(ob, md))
r = 0
try:
r = e.eval(md)
finally:
pop()
return r
_marker = object()
def mtime_match(ob, t, q):
mtime = getattr(ob, '_p_mtime', _marker)
if mtime is _marker():
return False
return q=='<' and (mtime < t) or (mtime > t)
def role_match(ob, permission, roles):
pr = []
while True:
p = getattr(ob, permission, _marker)
if p is not _marker:
if isinstance(p, list):
pr.append(p)
ob = aq_parent(ob)
if ob is not None:
continue
break
if isinstance(p, tuple):
pr.append(p)
break
if p is None:
pr.append(('Manager', 'Anonymous'))
break
ob = aq_parent(ob)
if ob is not None:
continue
break
for role in roles:
if role not in pr:
return False
return True
src/Products/ZCatalog/ZCatalogIndexes.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Virtual container for ZCatalog indexes.
"""
from AccessControl.class_init import InitializeClass
from AccessControl.SecurityInfo import ClassSecurityInfo
from AccessControl.Permissions import manage_zcatalog_indexes
from Acquisition import aq_base
from Acquisition import aq_parent
from Acquisition import Implicit
from App.special_dtml import DTMLFile
from OFS.Folder import Folder
from OFS.ObjectManager import IFAwareObjectManager
from OFS.SimpleItem import SimpleItem
from Persistence import Persistent
from Products.PluginIndexes.interfaces import IPluggableIndex
_marker = []
class ZCatalogIndexes(IFAwareObjectManager, Folder, Persistent, Implicit):
"""A mapping object, responding to getattr requests by looking up
the requested indexes in an object manager."""
# The interfaces we want to show up in our object manager
_product_interfaces = (IPluggableIndex, )
meta_type = "ZCatalogIndex"
manage_options = ()
security = ClassSecurityInfo()
security.declareObjectProtected(manage_zcatalog_indexes)
security.setPermissionDefault(manage_zcatalog_indexes, ('Manager', ))
security.declareProtected(manage_zcatalog_indexes, 'addIndexForm')
addIndexForm= DTMLFile('dtml/addIndexForm', globals())
# You no longer manage the Indexes here, they are managed from ZCatalog
def manage_main(self, REQUEST, RESPONSE):
"""Redirect to the parent where the management screen now lives"""
RESPONSE.redirect('../manage_catalogIndexes')
manage_workspace = manage_main
#
# Object Manager methods
#
# base accessors loop back through our dictionary interface
def _setOb(self, id, object):
indexes = aq_parent(self)._catalog.indexes
indexes[id] = object
aq_base(aq_parent(self))._indexes = indexes
def _delOb(self, id):
indexes = aq_parent(self)._catalog.indexes
del indexes[id]
aq_base(aq_parent(self))._indexes = indexes
def _getOb(self, id, default=_marker):
indexes = aq_parent(self)._catalog.indexes
if default is _marker:
return indexes.get(id)
return indexes.get(id, default)
security.declareProtected(manage_zcatalog_indexes, 'objectIds')
def objectIds(self, spec=None):
indexes = aq_parent(self)._catalog.indexes
if spec is not None:
if isinstance(spec, str):
spec = [spec]
result = []
for ob in indexes.keys():
o = indexes.get(ob)
meta = getattr(o, 'meta_type', None)
if meta is not None and meta in spec:
result.append(ob)
return result
return indexes.keys()
# Eat _setObject calls
def _setObject(self, id, object, roles=None, user=None, set_owner=1):
pass
#
# traversal
#
def __bobo_traverse__(self, REQUEST, name):
indexes = aq_parent(self)._catalog.indexes
o = indexes.get(name, None)
if o is not None:
if getattr(o, 'manage_workspace', None) is None:
o = OldCatalogWrapperObject(o)
return o.__of__(self)
return getattr(self, name)
InitializeClass(ZCatalogIndexes)
class OldCatalogWrapperObject(SimpleItem, Implicit):
manage_options= (
{'label': 'Settings',
'action': 'manage_main'},
)
manage_main = DTMLFile('dtml/manageOldindex', globals())
manage_main._setName('manage_main')
manage_workspace = manage_main
def __init__(self, o):
self.index = o
src/Products/ZCatalog/__init__.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""ZCatalog product"""
import ZCatalog
def initialize(context):
# Load a default map
from Products.ZCatalog.plan import PriorityMap
PriorityMap.load_default()
context.registerClass(
ZCatalog.ZCatalog,
permission='Add ZCatalogs',
constructors=(ZCatalog.manage_addZCatalogForm,
ZCatalog.manage_addZCatalog),
icon='www/ZCatalog.gif',
)
src/Products/ZCatalog/dtml/addIndexForm.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var "manage_form_title(this(), _, form_title='Add Index')">
<form action="manage_addIndex" method="post">
<input type=hidden name="type" value="&dtml-index_type;">
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Id
</div>
</td>
<td align="left" valign="top">
<input type="text" name="id" size="40" value="" />
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/ZCatalog/dtml/addZCatalog.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var "manage_form_title(this(), _, form_title='Add ZCatalog')">
<FORM ACTION="manage_addZCatalog" METHOD="POST">
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Id
</div>
</td>
<td align="left" valign="top">
<input type="text" name="id" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-optional">
Title
</div>
</td>
<td align="left" valign="top">
<input type="text" name="title" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/ZCatalog/dtml/catalogAdvanced.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<br />
<table width="100%" cellspacing="0" cellpadding="2" border="0">
<tr class="section-bar">
<td colspan="2" align="left">
<div class="form-label">
Catalog Maintenance
</div>
</td>
</tr>
<tr>
<td align="left" valign="top">
<p class="form-help"> Updating the catalog will update all catalog
records and remove invalid records. It does this by clearing all
indexes and re-cataloging all currently indexed objects.
</p>
</td>
<td align="right" valign="top">
<form action="&dtml-URL1;">
<input class="form-element" type="submit"
name="manage_catalogReindex:method" value=" Update Catalog ">
</form>
</td>
</tr>
<tr>
<td align="left" valign="top">
<p class="form-help">Clearing the catalog will remove all entries.
</p>
</td>
<td align="right" valign="top">
<form action="&dtml-URL1;">
<input class="form-element" type="submit"
name="manage_catalogClear:method" value=" Clear Catalog ">
</form>
</td>
</tr>
<tr>
<td align="left" valign="top">
<p class="form-help">Log progress of reindexing every N objects to the Zope logger (set to 0 to disable logging)
</p>
</td>
<td align="right" valign="top">
<form action="&dtml-URL1;">
<input type="text" name="pgthreshold:int" value="<dtml-var pgthreshold missing="0">">
<input class="form-element" type="submit"
name="manage_setProgress:method" value=" Change ">
</form>
</td>
</tr>
<tr>
<td>
</td>
</tr>
<tr class="section-bar">
<td colspan="2" align="left">
<div class="form-label">
Subtransactions
</div>
</td>
</tr>
<tr>
<td colspan="2" align="left" valign="top">
<p class="form-help"> Subtransactions allow Zope to commit small
parts of a transaction over a period of time instead of all at
once. For ZCatalog, this means using subtransactions can
signficantly reduce the memory requirements needed to index huge
amounts of text all at once. Currently, subtransactions are only
applied to text indexes.</p>
<p class="form-help"> If enabled, subtransactions will reduce the memory
requirements of ZCatalog, but <em>at the expense of speed</em>.
If you choose to enable subtransactions, you can adjust how often
ZCatalog commits a subtransactions by adjusting the
<em>threshold</em> below.</p>
</td>
</tr>
<tr>
<td align="left" valign="top">
<p>Subtransactions are
<dtml-if threshold>
<font color="green"><b>Enabled</b></font>
<dtml-else>
<font color="red"><b>Disabled</b></font>
</dtml-if></p>
</td>
<td align="right" valign="top">
<form action="&dtml-URL1;" method="POST">
<div class="form-element">
<dtml-if threshold>
<input class="form-element" type="submit"
name="manage_subbingToggle:method"
value="Disable" />
<dtml-else>
<input class="form-element" type="submit"
name="manage_subbingToggle:method"
value="Enable" />
</dtml-if>
</div>
</form>
</td>
</tr>
<dtml-if threshold>
<tr>
<td align="left" valign="top">
<p class="form-help">The Subtransaction threshold is the number of
objects cataloged
in the context of a single transaction that the catalog
will index before it commits a subtransaction. If this number
is low, the Catalog will take longer to index but consume less
memory. If this number is higher, the Catalog will index
quickly but consume much more memory.</p>
</td>
<td align="right" valign="top">
<form action="manage_edit" method=POST>
<div class="form-element">
<input name="threshold:int" value="&dtml-threshold;" />
<input type="submit" name="submit" value="Set Threshold">
</div>
</form>
</dtml-if>
</td>
</tr>
</table>
<dtml-var manage_page_footer>
src/Products/ZCatalog/dtml/catalogFind.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<P class="form-help">
Use this form to locate objects to be cataloged. Those objects which
are found will be automatically added to the catalog.
</p>
<FORM ACTION="manage_catalogFoundItems" METHOD="GET">
<TABLE>
<TR>
<TD ALIGN="LEFT" VALIGN="TOP">
<div class="form-label">
Find objects of type:
</div>
</TD>
<TD ALIGN="LEFT" VALIGN="TOP">
<div class="form-element">
<SELECT NAME="obj_metatypes:list" SIZE="4" MULTIPLE>
<OPTION VALUE="all" SELECTED> All types
<dtml-in all_meta_types mapping>
<OPTION VALUE="&dtml-name;"> &dtml-name;
</dtml-in>
</SELECT>
</div>
</TD>
</TR>
<TR>
<TD ALIGN="LEFT" VALIGN="TOP">
<div class="form-label">
with ids:
</div>
</TD>
<TD ALIGN="LEFT" VALIGN="TOP">
<INPUT TYPE="TEXT" NAME="obj_ids:tokens" SIZE="30">
</TD>
</TR>
<TR>
<TD ALIGN="LEFT" VALIGN="TOP">
<div class="form-label">
containing:
</div>
</TD>
<TD ALIGN="LEFT" VALIGN="TOP">
<INPUT TYPE="TEXT" NAME="obj_searchterm" SIZE="30">
</TD>
</TR>
<TR>
<TD ALIGN="LEFT" VALIGN="TOP">
<div class="form-label">
expr:
</div>
</TD>
<TD ALIGN="LEFT" VALIGN="TOP">
<TEXTAREA NAME="obj_expr" ROWS="4" COLS="30"></TEXTAREA>
</TD>
</TR>
<TR>
<TD ALIGN="LEFT" VALIGN="TOP">
<div class="form-label">
modified:
</div>
</TD>
<TD ALIGN="LEFT" VALIGN="TOP">
<div class="form-element">
<SELECT NAME="obj_mspec">
<OPTION VALUE="<"> before
<OPTION VALUE=">"> after
</SELECT>
</div>
<INPUT TYPE="TEXT" NAME="obj_mtime" SIZE="22">
</TD>
</TR>
<TR>
<TD ALIGN="LEFT" VALIGN="TOP">
<div class="form-label">
where the roles:
</div>
</TD>
<TD ALIGN="LEFT" VALIGN="TOP">
<div class="form-element">
<SELECT NAME="obj_roles:list" SIZE="3" MULTIPLE>
<dtml-in valid_roles>
<OPTION VALUE="&dtml-sequence-item;"> &dtml-sequence-item;
</dtml-in>
</SELECT>
</div>
</TD>
</TR>
<TR>
<TD ALIGN="LEFT" VALIGN="TOP">
<div class="form-label">
have permission:
</div>
</TD>
<TD ALIGN="LEFT" VALIGN="TOP">
<div class="form-element">
<SELECT NAME="obj_permission">
<dtml-in permission_settings mapping>
<OPTION VALUE="&dtml-name;"> &dtml-name;
</dtml-in>
</SELECT>
</div>
</TD>
</TR>
<INPUT TYPE="HIDDEN" NAME="search_sub:int" VALUE="1" CHECKED>
<TR>
<TD ALIGN="LEFT" VALIGN="TOP">
</TD>
<TD ALIGN="LEFT" VALIGN="TOP">
<div class="form-element">
<INPUT class="form-element" TYPE="SUBMIT" NAME="btn_submit"
VALUE="Find and Catalog">
</div>
</TD>
</TR>
</TABLE>
</FORM>
<dtml-var manage_page_footer>
src/Products/ZCatalog/dtml/catalogIndexes.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p class="form-help">
This list defines what indexes the Catalog will contain. When objects
get cataloged, the values of any attributes which match
an index in this list will get indexed.
</p>
<p class="form-help">
<b>
If you add indexes to a Catalog which contains indexed objects, you MUST
at the least re-index your newly added index. You may want to update the
whole Catalog.
</b>
</p>
<script type="text/javascript">
<!--
isSelected = false;
function toggleSelect() {
if (isSelected == false) {
for (i = 0; i < document.objectItems.length; i++)
document.objectItems.elements[i].checked = true ;
isSelected = true;
document.objectItems.selectButton.value = "Deselect All";
return isSelected;
}
else {
for (i = 0; i < document.objectItems.length; i++)
document.objectItems.elements[i].checked = false ;
isSelected = false;
document.objectItems.selectButton.value = "Select All";
return isSelected;
}
}
//-->
</script>
<dtml-unless skey><dtml-call expr="REQUEST.set('skey', 'id')"></dtml-unless>
<dtml-unless rkey><dtml-call expr="REQUEST.set('rkey', '')"></dtml-unless>
<dtml-with Indexes>
<!-- Add object widget -->
<br />
<dtml-if filtered_meta_types>
<table width="100%" cellspacing="0" cellpadding="0" border="0">
<tr>
<td align="left" valign="top">&nbsp;</td>
<td align="right" valign="top">
<div class="form-element">
<form action="&dtml-absolute_url;" method="get">
<dtml-if "_.len(filtered_meta_types) > 1">
<select class="form-element" name=":action"
onChange="location.href='&dtml-URL1;/'+this.options[this.selectedIndex].value">
<option value="manage_workspace" disabled>Select type to add...</option>
<dtml-in filtered_meta_types mapping sort=name>
<option value="&dtml.url_quote-action;">&dtml-name;</option>
</dtml-in>
</select>
<input class="form-element" type="submit" name="submit" value=" Add " />
<dtml-else>
<dtml-in filtered_meta_types mapping sort=name>
<input type="hidden" name=":method" value="&dtml.url_quote-action;" />
<input class="form-element" type="submit" name="submit" value=" Add &dtml-name;" />
</dtml-in>
</dtml-if>
</form>
</div>
</td>
</tr>
</table>
</dtml-if>
<form action="&dtml-URL1;/" name="objectItems" method="post">
<dtml-if objectItems>
<table width="100%" cellspacing="0" cellpadding="2" border="0">
<tr class="list-header">
<td>&nbsp;
</td>
<td width="30%" align="left"><div class="list-item"><a
href="./manage_catalogIndexes?skey=id<dtml-if
"rkey == ''">&rkey=id</dtml-if>"
onMouseOver="window.status='Sort objects by name'; return true"
onMouseOut="window.status=''; return true"><dtml-if
"skey == 'id' or rkey == 'id'"
><strong>Name</strong><dtml-else>Name</dtml-if></a></div>
</td>
<td width="30%" align="left"><div class="list-item"><a
href="./manage_catalogIndexes?skey=meta_type<dtml-if
"rkey == ''">&rkey=meta_type</dtml-if
>"
onMouseOver="window.status='Sort objects by type'; return true"
onMouseOut="window.status=''; return true"><dtml-if
"skey == 'meta_type' or rkey == 'meta_type'"
><strong>Index type</strong><dtml-else>Index type</dtml-if></a></div>
</td>
<td width="20%" align="left"><div class="list-item"><a
href="./manage_catalogIndexes?skey=indexSize<dtml-if
"rkey == ''">&rkey=indexSize</dtml-if
>"
onMouseOver="window.status='Sort objects by number of distinct values indexed'; return true"
onMouseOut="window.status=''; return true"><dtml-if
"skey == 'indexSize' or rkey == 'indexSize'"
><strong># distinct values</strong><dtml-else># distinct values</dtml-if></a></div>
</td>
<td width="20%" align="left"><div class="list-item"><a
href="./manage_catalogIndexes?skey=bobobase_modification_time<dtml-if
"rkey == ''">&rkey=bobobase_modification_time</dtml-if
>"
onMouseOver="window.status='Sort objects by modification time'; return true"
onMouseOut="window.status=''; return true"><dtml-if
"skey == 'bobobase_modification_time' or rkey == 'bobobase_modification_time'"
><strong>Last&nbsp;modified</strong><dtml-else>Last&nbsp;modified</dtml-if></a></div>
</td>
</tr>
<dtml-call "REQUEST.set('oldidx',0)">
<dtml-in objectItems sort_expr="skey" reverse_expr="rkey">
<dtml-if sequence-odd>
<tr class="row-normal">
<dtml-else>
<tr class="row-hilite">
</dtml-if>
<td align="left" valign="top" width="16">
<input type="checkbox" name="ids:list" value="&dtml-sequence-key;" />
</td>
<td align="left" valign="top">
<div class="list-item">
<a href="Indexes/&dtml.url_quote-sequence-key;/manage_workspace">
&dtml-sequence-key;
<dtml-try>
<dtml-let source_names="_['sequence-item'].getIndexSourceNames()">
<dtml-if expr="_.len(source_names) != 1
or source_names[0] != _['sequence-key']">
(indexed attributes: <dtml-var "', '.join(source_names)">)
</dtml-if>
</dtml-let>
<dtml-except>
<dtml-if title> (&dtml-title;) </dtml-if>
</dtml-try>
</a>
</div>
</td>
<dtml-with sequence-key>
<td>
<div class="list-item">
<dtml-var meta_type>
<dtml-if isDeprecatedIndex>
<dtml-call "REQUEST.set('oldidx',1)">
(pre-2.4 index)
</dtml-if>
</div>
</td>
<td>
<div class="list-item">
<dtml-var indexSize missing="n/a">
</div>
</td>
<td>
<div class="list-item">
<dtml-var bobobase_modification_time fmt="%Y-%m-%d %H:%M">
</div>
</td>
</dtml-with>
</tr>
</dtml-in>
</table>
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top" width="16"></td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="manage_delIndex:method" value="Remove index">
<input class="form-element" type="submit" name="manage_reindexIndex:method" value="Reindex">
<input class="form-element" type="submit" name="manage_clearIndex:method" value="Clear index">
<dtml-if oldidx>
<input class="form-element" type="submit" name="manage_convertIndex:method" value="Convert index">
</dtml-if>
<script type="text/javascript">
<!--
if (document.forms[0]) {
document.write('<input class="form-element" type="submit" name="selectButton" value="Select All" onClick="toggleSelect(); return false">')
}
//-->
</script>
</div>
</td>
</tr>
</table>
<dtml-else>
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td>
<div class="std-text">
<em>There are currently no indexes</em>
<br /><br />
</div>
</td>
</tr>
</table>
</dtml-if>
</form>
<dtml-if update_menu>
<script type="text/javascript">
<!--
window.parent.update_menu();
//-->
</script>
</dtml-if>
</dtml-with>
<dtml-var manage_page_footer>
src/Products/ZCatalog/dtml/catalogObjectInformation.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<table width="100%" borders="0" cellspacing="2" cellpadding="0">
<tr bgcolor="#000000">
<td colspan="2">&nbsp;</td>
</tr>
<tr>
<td colspan="2">&nbsp;</td>
</tr>
<tr class="location-bar">
<td colspan="2" align="left">
<div class="std-text">
<strong>Catalog record at <dtml-var expr="getpath(_.int(rid))" html_quote></strong>
</div>
</td>
</tr>
<tr>
<td colspan="2" align="left">
<p class="form-help">
The goal of this page is to provide basic debugging information
about what is in the Catalog on a specific object. Listed below is
all the information that the Catalog currently contains for the
above specified object. This information should match what is
currently in the instance of that object.
</td>
</tr>
</table>
<br />
<table width="100%" borders="0" cellspacing="2" cellpadding="0">
<tr class="section-bar">
<td colspan="3" align="left">
<div class="form-label">Metadata Contents</div>
</td>
</tr>
<tr>
<td colspan="3" align="left">
<p class="form-help">Metadata is the information that the Catalog
keeps inside of its internal structure so that it can answer
questions quickly. This is then returned in the "brain" that the
Catalog gives back during searches.</p><br />
</td>
</tr>
<dtml-in expr="getMetadataForRID(_.int(rid)).items()">
<dtml-if name="sequence-start">
<tr class="list-header">
<td align="left" width="5%" bgcolor="#ffffff">&nbsp;</td>
<td align="left" width="25%" valign="top" class="list-item">Key</td>
<td align="left" width="70%" valign="top" class="list-item">Value</td>
</dtml-if>
<dtml-if name="sequence-odd"><tr class="row-hilite">
<dtml-else><tr></dtml-if>
<td width="32" bgcolor="#ffffff">&nbsp;</td>
<td align="left" valign="top" class="form-element">
&dtml-sequence-key;
</td>
<Td align="left" valign="top" class="form-element">
&dtml-sequence-item;
</td>
</tr>
</dtml-in>
</table>
<br />
<table width="100%" borders="0" cellspacing="2" cellpadding="0">
<tr class="section-bar">
<td colspan="3" align="left">
<div class="form-label">Index Contents</div>
</td>
</tr>
<tr>
<td colspan="3" align="left">
<p class="form-help">The following table gives information that is
contained in the various indexes of the Catalog. In the case of
Keyword or Text indexes, the results are returned as a tuple, and will
show as '(one, two, three)', rather than in a more normal way.</p><br />
</td>
</tr>
<dtml-in expr="getIndexDataForRID(_.int(rid)).items()">
<dtml-if name="sequence-start">
<tr class="list-header">
<td align="left" width="5%" bgcolor="#ffffff">&nbsp;</td>
<td align="left" width="25%" valign="top" class="list-item">Key</td>
<td align="left" width="70%" valign="top" class="list-item">Value</td>
</dtml-if>
<dtml-if name="sequence-odd"><tr class="row-hilite">
<dtml-else><tr></dtml-if>
<td width="32" bgcolor="#ffffff">&nbsp;</td>
<td align="left" valign="top" class="form-element">
&dtml-sequence-key;
</td>
<td align="left" valign="top" class="form-element">
&dtml-sequence-item;
</td>
</tr>
</dtml-in>
</table>
<dtml-var manage_page_footer>
src/Products/ZCatalog/dtml/catalogPlan.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p class="form-help">
The <strong>query plan</strong> shows the actual query plan of the
current process.
</p>
<textarea name="queryplan" cols="70" rows="25" readonly="readonly">
&dtml-getCatalogPlan;
</textarea>
<dtml-var manage_page_footer>
src/Products/ZCatalog/dtml/catalogReport.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p class="form-help">
The <strong>query report</strong> shows catalog queries that
perform slowly.
</p>
<table width="100%" cellspacing="0" cellpadding="2" border="0">
<tr class="list-header" >
<td align="left" valign="top">
<div class="list-nav">
Mean duration&nbsp;[ms]
</div>
</td>
<td align="left" valign="top">
<div class="list-nav">
Hits
</div>
</td>
<td align="left" valign="top">
<div class="list-nav">
Query key
</div>
</td>
<td align="left" valign="top">
<div class="list-nav">
Recent
</div>
</td>
</tr>
<dtml-if getCatalogReport>
<dtml-in getCatalogReport mapping>
<dtml-if sequence-odd>
<tr class="row-normal">
<dtml-else>
<tr class="row-hilite">
</dtml-if>
<td align="left" valign="top">
<div class="list-item">
<dtml-var expr="'%3.2f' % duration">
</div>
</td>
<td align="left" valign="top">
<div class="list-item">
&dtml-counter;
</div>
</td>
<td align="left" valign="top">
<div class="list-item">
&dtml-query;
</div>
</td>
<td align="left" valign="top">
<div class="list-item">
<dtml-var expr="'%3.2f' % last['duration']">ms
[<dtml-in expr="last['details']" sort mapping>
&dtml-id;:
<dtml-var expr="'%3.2f' % duration">ms /
&dtml-length; objects,
</dtml-in>]
</div>
</td>
</tr>
</dtml-in>
<tr>
<td colspan="2" align="left" valign="top">
<p class="form-help">Resetting the catalog report will reinitialize the report log.</p>
</td>
<td colspan="2" align="right" valign="top">
<form action="manage_resetCatalogReport" method=POST>
<div class="form-element">
<input class="form-element" type="submit" value="Reset Report">
</div>
</form>
</td>
</tr>
<dtml-else>
<tr>
<td colspan="4" >
<div class="list-item">
Report is empty.
</div>
</td>
</tr>
</dtml-if>
</table>
<form action="manage_editCatalogReport" method="post">
<table width="100%" style="padding-top:1em;" cellspacing="0" cellpadding="2" border="0">
<tr class="section-bar">
<td colspan="3" align="left">
<div class="form-label">
Settings
</div>
</td>
</tr>
<tr>
<td align="right" valign="middle">
<div class="list-item">
Threshold (in seconds)
</div>
</td>
<td align="left" valign="middle">
<div class="form-element">
<input name="long_query_time:float" value="&dtml-long_query_time;" />
</div>
</td>
<td align="left" valign="middle">
<p class="form-help">Only queries whose execution
takes longer than the configured threshold are considered
being slow. (Default 0.1 seconds).</p>
</tr>
</table>
<input class="form-element" type="submit" value="Apply settings">
</form>
<dtml-var manage_page_footer>
src/Products/ZCatalog/dtml/catalogSchema.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p class="form-help">
This list defines what per object meta data the Catalog will store.
When objects get cataloged, the values of any attributes they may have
which match a name in this list will get stored in a table in the
Catalog. The Catalog then uses this information to create result
objects that are returned whenever the catalog is searched. It is
important to understand that when the Catalog is searched, it returns
a list of result objects, <i>not the cataloged objects themselves</i>,
so if you want to use the value of an object's attribute in the result
of a search, that attribute must be in this list
</p>
<p class="form-help">
It is generally a good idea to keep this list lightweight. It is
useful, for example, to keep the 'summary' meta data of a text
document (like the first 200 characters) but <i>not</i> the text
content in it's entirety (it is useful in this example to <i>index</i>
the text contents, which is configured in the <b>Indexes</b> View
tab). This way, the summary data may be shown in the search results.
</p>
<form action="&dtml-URL1;">
<table cellspacing="0" cellpadding="2" border="0">
<dtml-in schema sort=sequence-item>
<tr>
<td align="left" valign="top">
<input type="checkbox" name="names:list" value="&dtml-sequence-item;" />
</td>
<td align="left" valign="top">
<div class="form-text">
&dtml-sequence-item;
</div>
</td>
</tr>
<dtml-if sequence-end>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="manage_delColumn:method"
value="Delete" />
</div>
</td>
</tr>
</dtml-if>
<dtml-else>
<tr>
<td></td>
<td><em class="std-text">There are currently no metadata elements.</em></td>
</tr>
</dtml-in>
</table>
<br />
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Add Metadata
</div>
</td>
<td align="left" valign="top">
<input type="text" name="name" size="20" />
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="manage_addColumn:method"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/ZCatalog/dtml/catalogStatus.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p class="form-help"> Subtransactions allow Zope to commit small
parts of a transaction over a period of time instead of all at once.
For
ZCatalog, this means using subtransactions can signficantly
reduce the memory requirements needed to index huge amounts of
text all at once.</p>
<p class="form-help"> If enabled, subtransactions will reduce the memory
requirements of ZCatalog, but <em>at the expense of speed</em>.
If you choose to enable subtransactions, you can adjust how often
ZCatalog commits a subtransactions by adjusting the
<em>threshold</em> below.</p>
<h3>Subtransactions are
<dtml-if threshold>
<font color="green"><b>Enabled</b></font>
<dtml-else>
<font color="red"><b>Disabled</b></font>
</dtml-if></h3>
<form action="&dtml-URL1;" method="POST">
<div class="form-element">
<dtml-if threshold>
<input class="form-element" type="submit"
name="manage_subbingToggle:method"
value="Disable" /> Subtransactions
<dtml-else>
<input class="form-element" type="submit"
name="manage_subbingToggle:method"
value="Enable" /> Subtransactions
</dtml-if>
</div>
</form>
<form action="manage_edit" method=POST>
<dtml-if threshold>
<p class="form-help">The Subtransaction threshold is the number of words the catalog
will index before it commits a subtransaction. If this number
is low, the Catalog will take longer to index but consume less
memory. If this number is higher, the Catalog will index
quickly but consume much more memory.</p>
Subtransaction threshold: <input name="threshold:int" value="&dtml-threshold;" />
<br>
<div class="form-element">
<input type="submit" name="submit" value="Save Changes">
</div>
</dtml-if>
</form>
<hr width=75%>
<h3>Index Status</h3>
<ul>
<dtml-in index_objects sort=id>
<li>
<dtml-var "_.len(_['sequence-item'])">
object are indexed in <b><dtml-var "_['sequence-item'].id" html_quote></b>
</li>
</dtml-in>
</ul>
<dtml-var manage_page_footer>
src/Products/ZCatalog/dtml/catalogView.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<dtml-let filterpath="REQUEST.get('filterpath', '/')"
results="searchResults(path=filterpath)">
<dtml-if results>
<script type="text/javascript">
<!--
isSelected = false;
function toggleSelect() {
if (isSelected == false) {
for (i = 0; i < document.objectItems.length; i++)
document.objectItems.elements[i].checked = true ;
isSelected = true;
document.objectItems.selectButton.value = "Deselect All";
return isSelected;
}
else {
for (i = 0; i < document.objectItems.length; i++)
document.objectItems.elements[i].checked = false ;
isSelected = false;
document.objectItems.selectButton.value = "Select All";
return isSelected;
}
}
//-->
</script>
<h1 class="form-label section-bar">Path filter</h1>
<dtml-if "'path' in this().Indexes.objectIds()">
<form action="&dtml-URL;">
<p class="form-text">
Path: <input type="text" name="filterpath" value="&dtml-filterpath;"/> <input type="submit" value="Set Filter"/>
</p>
</form>
<dtml-else>
<p class="form-text">
The path filter is <span style="color:red;">disabled</span>. To enable the path filter, add a PathIndex called "path" to this catalog.
</p>
</dtml-if>
<h1 class="form-label section-bar">Objects in this catalog</h1>
<form action="&dtml-URL1;" name="objectItems">
<p class="form-text">
The catalog "&dtml-id;" contains <dtml-var results fmt=collection-length thousands_commas> record(s) in the path "&dtml-filterpath;".
</p>
<div class="form-text">
<dtml-in results previous size=20 start=query_start >
<a href="&dtml-URL;?query_start=&dtml-previous-sequence-start-number;&filterpath=&dtml-filterpath;">
[Previous <dtml-var previous-sequence-size> entries]
</a>
</dtml-in>
<dtml-in results next size=20 start=query_start >
<a href="&dtml-URL;?query_start=&dtml-next-sequence-start-number;&filterpath=&dtml-filterpath;">
[Next <dtml-var next-sequence-size> entries]
</a>
</dtml-in>
</div>
<table width="100%" cellspacing="0" cellpadding="2" border="0">
<dtml-in results size=20 start=query_start >
<dtml-if name="sequence-start">
<tr class="list-header">
<td width="5%" align="right" colspan="2" valign="top">&nbsp;</td>
<td width="80%" align="left" valign="top">
<div class="list-item">Object Identifier</div></td>
<td width="15%" align="left" valign="top">
<div class="list-item">Type</div></td>
</tr>
</dtml-if>
<dtml-if name="sequence-odd"><tr class="row-normal">
<dtml-else><tr class="row-hilite"></dtml-if>
<td align="right" valign="top">
<input type="checkbox" NAME="urls:list" VALUE="&dtml-getPath;">
</td>
<td align="left" valign="top">&nbsp;</td>
<td align="left" valign="top">
<div class="form-text">
<a href="&dtml-URL1;/manage_objectInformation?rid=&dtml-getRID;"
target="_objectinfo_&dtml-getRID;">&dtml-getPath;</a>
</div>
</td>
<td align="left" valign="top">
<div class="form-text">
<dtml-if expr="has_key('meta_type') and meta_type">
<dtml-var name="meta_type" size="15" html_quote>
<dtml-else>
<i>Unknown</i>
</dtml-if>
</div>
</td>
</tr>
</dtml-in>
</table>
<div class="form-element">
<input class="form-element" type="submit" value=" Remove "
name="manage_uncatalogObject:method">
<input class="form-element" type="submit" value=" Update "
name="manage_catalogObject:method">
<script type="text/javascript">
<!--
if (document.forms[0]) {
document.write('<input class="form-element" type="submit" name="selectButton" value="Select All" onClick="toggleSelect(); return false">')
}
//-->
</script>
</div>
</form>
<dtml-else>
<p class="form-text">
There are no objects in the Catalog.
</p>
</dtml-if>
</dtml-let>
<dtml-var manage_page_footer>
src/Products/ZCatalog/dtml/editCatalogerForm.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<form action="manage_editCataloger">
<p class="form-help">
This element controls which Catalog this object will try index and
unindex itself to.
</p>
<p>
<span class="form-label">
Use Catalog:
</span>
<input name="default" value="&dtml-default_catalog;">
<br>
<div class="form-element">
<input class="form-element" type="submit" value="Save Changes">
</div>
</form>
<dtml-var manage_page_footer>
src/Products/ZCatalog/interfaces.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2005 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""ZCatalog interfaces.
"""
from zope.interface import Interface
class IZCatalog(Interface):
"""ZCatalog object
A ZCatalog contains arbitrary index like references to Zope
objects. ZCatalog's can index object attribute using a variety
of "plug-in" index types.
Several index types are included, and others may be added.
Text -- Text indexes index textual content. The index can be
used to search for objects containing certain words.
Field -- Field indexes index atomic values. The index can be
used to search for objects that have certain properties.
Keyword -- Keyword indexes index sequences of values. The index
can be used to search for objects that match one or more of the
search terms.
Path -- Path indexes index URI paths. They allow you to find objects
based on their placement in a hierarchy.
Date -- Date indexes index date and type data. They are a type of field
index specifically optimized for indexing dates.
Date Range -- Date range indexes index time intervals. They are designed
for efficient searching of dates falling between two boundaries
(such as effective / expiration dates).
Topic -- Topic indexes store prefiltered sets of documents. They are used
to optimize complex queries into a single fast query by prefiltering
documents by an expression
The ZCatalog can maintain a table of extra data about cataloged
objects. This information can be used on search result pages to
show information about a search result.
The meta-data table schema is used to build the schema for
ZCatalog Result objects. The objects have the same attributes
as the column of the meta-data table.
ZCatalog does not store references to the objects themselves, but
rather to a unique identifier that defines how to get to the
object. In Zope, this unique identifier is the object's relative
path to the ZCatalog (since two Zope objects cannot have the same
URL, this is an excellent unique qualifier in Zope).
"""
def catalog_object(obj, uid, idxs=None, update_metadata=1):
"""Catalogs the object 'obj' with the unique identifier 'uid'.
The uid must be a physical path, either absolute or relative to
the catalog.
If provided, idxs specifies the names of indexes to update.
If update_metadata is specified (the default), the object's metadata
is updated. If it is not, the metadata is left untouched. This
flag has no effect if the object is not yet cataloged (metadata
is always added for new objects).
"""
def uncatalog_object(uid):
"""Uncatalogs the object with the unique identifier 'uid'.
The uid must be a physical path, either absolute or relative to
the catalog.
"""
def uniqueValuesFor(name):
"""returns the unique values for a given FieldIndex named 'name'.
"""
def getpath(rid):
"""Return the path to a cataloged object given a 'data_record_id_'
"""
def getrid(rid):
"""Return the 'data_record_id_' to a cataloged object given a path
"""
def getobject(rid, REQUEST=None):
"""Return a cataloged object given a 'data_record_id_'
"""
def schema():
"""Get the meta-data schema
Returns a sequence of names that correspond to columns in the
meta-data table.
"""
def indexes():
"""Returns a sequence of names that correspond to indexes.
"""
def index_objects():
"""Returns a sequence of actual index objects.
NOTE: This returns unwrapped indexes! You should probably use
getIndexObjects instead. Some indexes expect to be wrapped.
"""
def getIndexObjects():
"""Returns a list of acquisition wrapped index objects
"""
def searchResults(REQUEST=None, **kw):
"""Search the catalog.
Search terms can be passed in the REQUEST or as keyword
arguments.
Search queries consist of a mapping of index names to search
parameters. You can either pass a mapping to searchResults as
the variable 'REQUEST' or you can use index names and search
parameters as keyword arguments to the method, in other words::
searchResults(title='Elvis Exposed',
author='The Great Elvonso')
is the same as::
searchResults({'title' : 'Elvis Exposed',
'author : 'The Great Elvonso'})
In these examples, 'title' and 'author' are indexes. This
query will return any objects that have the title *Elvis
Exposed* AND also are authored by *The Great Elvonso*. Terms
that are passed as keys and values in a searchResults() call
are implicitly ANDed together. To OR two search results, call
searchResults() twice and add concatenate the results like this::
results = ( searchResults(title='Elvis Exposed') +
searchResults(author='The Great Elvonso') )
This will return all objects that have the specified title OR
the specified author.
There are some special index names you can pass to change the
behavior of the search query:
sort_on -- This parameters specifies which index to sort the
results on.
sort_order -- You can specify 'reverse' or 'descending'.
Default behavior is to sort ascending.
sort_limit -- An optimization hint to tell the catalog how many
results you are really interested in. See the limit argument
to the search method for more details.
There are some rules to consider when querying this method:
- an empty query mapping (or a bogus REQUEST) returns all
items in the catalog.
- results from a query involving only field/keyword
indexes, e.g. {'id':'foo'} and no 'sort_on' will be
returned unsorted.
- results from a complex query involving a field/keyword
index *and* a text index,
e.g. {'id':'foo','PrincipiaSearchSource':'bar'} and no
'sort_on' will be returned unsorted.
- results from a simple text index query
e.g.{'PrincipiaSearchSource':'foo'} will be returned
sorted in descending order by 'score'. A text index
cannot beused as a 'sort_on' parameter, and attempting
to do so will raise an error.
Depending on the type of index you are querying, you may be
able to provide more advanced search parameters that can
specify range searches or wildcards. These features are
documented in The Zope Book.
"""
def __call__(REQUEST=None, **kw):
"""Search the catalog, the same way as 'searchResults'.
"""
def search(query_request, sort_index=None, reverse=0, limit=None, merge=1):
"""Programmatic search interface, use for searching the catalog from
scripts.
query_request -- Dictionary containing catalog query. This uses the
same format as searchResults.
sort_index -- Name of sort index
reverse -- Boolean, reverse sort order (defaults to false)
limit -- Limit sorted result count to the n best records. This is an
optimization hint used in conjunction with a sort_index. If possible
ZCatalog will use a different sort algorithm that uses much less memory
and scales better then a full sort. The actual number of records
returned is not guaranteed to be <= limit. You still need to apply the
same batching to the results. Since the len() of the results will no
longer be the actual result count, you can use the
"actual_result_count" attribute of the lazy result object instead to
determine the size of the full result set.
merge -- Return merged, lazy results (like searchResults) or raw
results for later merging. This can be used to perform multiple
queries (even across catalogs) and merge and sort the combined results.
"""
def refreshCatalog(clear=0, pghandler=None):
"""Reindex every object we can find, removing the unreachable
ones from the index.
clear -- values: 1|0 clear the catalog before reindexing
pghandler -- optional Progresshandler as defined in ProgressHandler.py
(see also README.txt)
"""
def reindexIndex(name, REQUEST, pghandler=None):
"""Reindex a single index.
name -- id of index
REQUEST -- REQUEST object
pghandler -- optional Progresshandler as defined in ProgressHandler.py
(see also README.txt)
"""
# This should inherit from an IRecord interface, if there ever is one.
class ICatalogBrain(Interface):
"""Catalog brain that handles looking up attributes as
required, and provides just enough smarts to let us get the URL, path,
and cataloged object without having to ask the catalog directly.
"""
def has_key(key):
"""Record has this field"""
def __contains__(self, name):
"""Record has this field"""
def getPath():
"""Get the physical path for this record"""
def getURL(relative=0):
"""Generate a URL for this record"""
def _unrestrictedGetObject():
"""Return the object for this record
Same as getObject, but does not do security checks.
"""
def getObject():
"""Return the object for this record
Will return None if the object cannot be found via its cataloged path
(i.e., it was deleted or moved without recataloging), or if the user is
not authorized to access the object.
"""
def getRID():
"""Return the record ID for this object."""
class IProgressHandler(Interface):
""" A handler to log progress informations for long running
operations.
"""
def init(ident, max):
""" Called at the start of the long running process.
'ident' -- a string identifying the operation
'max' -- maximum number of objects to be processed (int)
"""
def info(text):
""" Log some 'text'"""
def finish():
""" Called up termination """
def report(current, *args, **kw):
""" Called for every iteration.
'current' -- an integer representing the number of objects
processed so far.
"""
def output(text):
""" Log 'text' to some output channel """
src/Products/ZCatalog/plan.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2010 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import time
from collections import namedtuple
from logging import getLogger
from os import environ
from thread import allocate_lock
from Acquisition import aq_base
from Acquisition import aq_parent
from Products.PluginIndexes.interfaces import IUniqueValueIndex
from zope.dottedname.resolve import resolve
MAX_DISTINCT_VALUES = 10
REFRESH_RATE = 100
Duration = namedtuple('Duration', ['start', 'end'])
IndexMeasurement = namedtuple('IndexMeasurement',
['name', 'duration', 'num', 'limit'])
Benchmark = namedtuple('Benchmark', ['num', 'duration', 'hits', 'limit'])
RecentQuery = namedtuple('RecentQuery', ['duration', 'details'])
Report = namedtuple('Report', ['hits', 'duration', 'last'])
logger = getLogger('Products.ZCatalog')
class NestedDict(object):
"""Holds a structure of two nested dicts."""
@classmethod
def get(cls, key):
outer = cls.value.get(key, None)
if outer is None:
cls.set(key, {})
outer = cls.value[key]
return outer
@classmethod
def set(cls, key, value):
with cls.lock:
cls.value[key] = value
@classmethod
def clear(cls):
with cls.lock:
cls.value = {}
@classmethod
def get_entry(cls, key, key2):
outer = cls.get(key)
inner = outer.get(key2, None)
if inner is None:
cls.set_entry(key, key2, {})
inner = outer.get(key2)
return inner
@classmethod
def set_entry(cls, key, key2, value):
outer = cls.get(key)
with cls.lock:
outer[key2] = value
@classmethod
def clear_entry(cls, key):
cls.set(key, {})
class PriorityMap(NestedDict):
"""This holds a structure of nested dicts.
The outer dict is a mapping of catalog id to plans. The inner dict holds
a query key to Benchmark mapping.
"""
lock = allocate_lock()
value = {}
@classmethod
def get_value(cls):
return cls.value.copy()
@classmethod
def load_default(cls):
location = environ.get('ZCATALOGQUERYPLAN')
if location:
try:
pmap = resolve(location)
logger.info('loaded priority %d map(s) from %s',
len(pmap), location)
# Convert the simple benchmark tuples to namedtuples
new_plan = {}
for cid, plan in pmap.items():
new_plan[cid] = {}
for querykey, details in plan.items():
new_plan[cid][querykey] = {}
for indexname, benchmark in details.items():
new_plan[cid][querykey][indexname] = \
Benchmark(*benchmark)
with cls.lock:
cls.value = new_plan
except ImportError:
logger.warning('could not load priority map from %s', location)
class Reports(NestedDict):
"""This holds a structure of nested dicts.
The outer dict is a mapping of catalog id to reports. The inner dict holds
a query key to Report mapping.
"""
lock = allocate_lock()
value = {}
class ValueIndexes(object):
"""Holds a set of index names considered to have an uneven value
distribution.
"""
lock = allocate_lock()
value = frozenset()
@classmethod
def get(cls):
return cls.value
@classmethod
def set(cls, value):
value = frozenset(value)
with cls.lock:
cls.value = value
@classmethod
def clear(cls):
cls.set(frozenset())
@classmethod
def determine(cls, indexes):
# This function determines all indexes whose values should be respected
# in the report key. The number of unique values for the index needs to
# be lower than the MAX_DISTINCT_VALUES watermark.
# TODO: Ideally who would only consider those indexes with a small
# number of unique values, where the number of items for each value
# differs a lot. If the number of items per value is similar, the
# duration of a query is likely similar as well.
value_indexes = cls.get()
if value_indexes:
# Calculating all the value indexes is quite slow, so we do this
# once for the first query. Since this is an optimization only,
# slightly outdated results based on index changes in the running
# process can be ignored.
return value_indexes
value_indexes = set()
for name, index in indexes.items():
if IUniqueValueIndex.providedBy(index):
values = index.uniqueValues()
if values and len(list(values)) < MAX_DISTINCT_VALUES:
# Only consider indexes which actually return a number
# greater than zero
value_indexes.add(name)
cls.set(value_indexes)
return value_indexes
def make_key(catalog, query):
if not query:
return None
indexes = catalog.indexes
valueindexes = ValueIndexes.determine(indexes)
key = keys = query.keys()
values = [name for name in keys if name in valueindexes]
if values:
# If we have indexes whose values should be considered, we first
# preserve all normal indexes and then add the keys whose values
# matter including their value into the key
key = [name for name in keys if name not in values]
for name in values:
v = query.get(name, [])
if isinstance(v, (tuple, list)):
v = list(v)
v.sort()
# We need to make sure the key is immutable, repr() is an easy way
# to do this without imposing restrictions on the types of values
key.append((name, repr(v)))
key = tuple(sorted(key))
return key
class CatalogPlan(object):
"""Catalog plan class to measure and identify catalog queries and plan
their execution.
"""
def __init__(self, catalog, query=None, threshold=0.1):
self.catalog = catalog
self.query = query
self.key = make_key(catalog, query)
self.benchmark = {}
self.threshold = threshold
self.cid = self.get_id()
self.init_timer()
def get_id(self):
parent = aq_parent(self.catalog)
path = getattr(aq_base(parent), 'getPhysicalPath', None)
if path is None:
path = ('', 'NonPersistentCatalog')
else:
path = tuple(parent.getPhysicalPath())
return path
def init_timer(self):
self.res = []
self.start_time = None
self.interim = {}
self.stop_time = None
self.duration = None
def plan(self):
benchmark = PriorityMap.get_entry(self.cid, self.key)
if not benchmark:
return None
# sort indexes on (mean result length, mean search time)
ranking = [((value.limit, value.num, value.duration), name)
for name, value in benchmark.items()]
ranking.sort()
return [r[1] for r in ranking]
def start(self):
self.init_timer()
self.start_time = time.time()
def start_split(self, name):
self.interim[name] = Duration(time.time(), None)
def stop_split(self, name, result=None, limit=False):
current = time.time()
start_time, stop_time = self.interim.get(name, Duration(None, None))
length = 0
if result is not None:
# TODO: calculating the length can be expensive
length = len(result)
self.interim[name] = Duration(start_time, current)
dt = current - start_time
self.res.append(IndexMeasurement(
name=name, duration=dt, num=length, limit=limit))
if name == 'sort_on':
# sort_on isn't an index. We only do time reporting on it
return
# remember index's hits, search time and calls
benchmark = self.benchmark
if name not in benchmark:
benchmark[name] = Benchmark(num=length, duration=dt,
hits=1, limit=limit)
else:
num, duration, hits, limit = benchmark[name]
num = int(((num * hits) + length) / float(hits + 1))
duration = ((duration * hits) + dt) / float(hits + 1)
# reset adaption
if hits % REFRESH_RATE == 0:
hits = 0
hits += 1
benchmark[name] = Benchmark(num, duration, hits, limit)
def stop(self):
self.end_time = time.time()
self.duration = self.end_time - self.start_time
# Make absolutely sure we never omit query keys from the plan
for key in self.query.keys():
if key not in self.benchmark.keys():
self.benchmark[key] = Benchmark(0, 0, 0, False)
PriorityMap.set_entry(self.cid, self.key, self.benchmark)
self.log()
def log(self):
# result of stopwatch
total = self.duration
if total < self.threshold:
return
key = self.key
recent = RecentQuery(duration=total, details=self.res)
previous = Reports.get_entry(self.cid, key)
if previous:
counter, mean, last = previous
mean = (mean * counter + total) / float(counter + 1)
Reports.set_entry(self.cid, key, Report(counter + 1, mean, recent))
else:
Reports.set_entry(self.cid, key, Report(1, total, recent))
def reset(self):
Reports.clear_entry(self.cid)
def report(self):
"""Returns a statistic report of catalog queries as list of dicts.
The duration is provided in millisecond.
"""
rval = []
for key, report in Reports.get(self.cid).items():
last = report.last
info = {
'query': key,
'counter': report.hits,
'duration': report.duration * 1000,
'last': {'duration': last.duration * 1000,
'details': [dict(id=d.name,
duration=d.duration * 1000,
length=d.num)
for d in last.details],
},
}
rval.append(info)
return rval
# Make sure we provide test isolation
from zope.testing.cleanup import addCleanUp
addCleanUp(PriorityMap.clear)
addCleanUp(Reports.clear)
addCleanUp(ValueIndexes.clear)
del addCleanUp
src/Products/ZCatalog/tests/test_brains.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Unittests for Catalog brains
"""
import unittest
import Acquisition
from Acquisition import aq_base
from zExceptions import Unauthorized
from ZODB.POSException import ConflictError
_marker = object()
class Happy(Acquisition.Implicit):
"""Happy content"""
def __init__(self, id):
self.id = id
def check(self):
pass
class Secret(Happy):
"""Object that raises Unauthorized when accessed"""
def check(self):
raise Unauthorized
class Conflicter(Happy):
"""Object that raises ConflictError when accessed"""
def check(self):
raise ConflictError
class DummyRequest(object):
def physicalPathToURL(self, path, relative=False):
if not relative:
path = 'http://superbad.com' + path
return path
class DummyCatalog(Acquisition.Implicit):
_objs = {'/happy': Happy('happy'),
'/secret': Secret('secret'),
'/conflicter': Conflicter('conflicter')}
_paths = _objs.keys() + ['/zonked']
_paths.sort()
# This is sooooo ugly
def unrestrictedTraverse(self, path, default=None):
assert path in ['', ('', ), ['']], path
return self
def restrictedTraverse(self, path, default=_marker):
if not path.startswith('/'):
path = '/'+path
try:
ob = self._objs[path].__of__(self)
ob.check()
return ob
except KeyError:
if default is not _marker:
return default
raise
def getpath(self, rid):
return self._paths[rid]
def getobject(self, rid):
return self.restrictedTraverse(self._paths[rid])
def resolve_url(self, path, REQUEST):
# strip server part
path = path[path.find('/', path.find('//') + 1):]
return self.restrictedTraverse(path)
class ConflictingCatalog(DummyCatalog):
def getpath(self, rid):
raise ConflictError
class TestBrains(unittest.TestCase):
def setUp(self):
self.cat = DummyCatalog()
self.cat.REQUEST = DummyRequest()
def _makeBrain(self, rid):
from Products.ZCatalog.CatalogBrains import AbstractCatalogBrain
class Brain(AbstractCatalogBrain):
__record_schema__ = {'test_field': 0, 'data_record_id_': 1}
return Brain(('test', rid)).__of__(self.cat)
def testHasKey(self):
b = self._makeBrain(1)
self.assertTrue('test_field' in b)
self.assertTrue('data_record_id_' in b)
self.assertFalse('godel' in b)
def testGetPath(self):
b = [self._makeBrain(rid) for rid in range(3)]
self.assertEqual(b[0].getPath(), '/conflicter')
self.assertEqual(b[1].getPath(), '/happy')
self.assertEqual(b[2].getPath(), '/secret')
def testGetPathPropagatesConflictErrors(self):
self.cat = ConflictingCatalog()
b = self._makeBrain(0)
self.assertRaises(ConflictError, b.getPath)
def testGetURL(self):
b = self._makeBrain(0)
self.assertEqual(b.getURL(), 'http://superbad.com/conflicter')
def testGetRID(self):
b = self._makeBrain(42)
self.assertEqual(b.getRID(), 42)
def testGetObjectHappy(self):
b = self._makeBrain(1)
self.assertEqual(b.getPath(), '/happy')
self.assertTrue(aq_base(b.getObject()) is
aq_base(self.cat.getobject(1)))
def testGetObjectPropagatesConflictErrors(self):
b = self._makeBrain(0)
self.assertEqual(b.getPath(), '/conflicter')
self.assertRaises(ConflictError, b.getObject)
def testGetObjectRaisesUnauthorized(self):
from zExceptions import Unauthorized
b = self._makeBrain(2)
self.assertEqual(b.getPath(), '/secret')
self.assertRaises(Unauthorized, b.getObject)
def testGetObjectRaisesNotFoundForMissing(self):
from zExceptions import NotFound
b = self._makeBrain(3)
self.assertEqual(b.getPath(), '/zonked')
self.assertRaises(KeyError, self.cat.getobject, 3)
self.assertRaises((NotFound, AttributeError, KeyError), b.getObject)
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestBrains))
return suite
src/Products/ZCatalog/tests/test_catalog.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
""" Unittests for Catalog.
"""
import unittest
from Testing.ZopeTestCase.warnhook import WarningsHook
import Zope2
Zope2.startup()
from itertools import chain
import random
import ExtensionClass
import OFS.Application
from Products.PluginIndexes.FieldIndex.FieldIndex import FieldIndex
from Products.PluginIndexes.KeywordIndex.KeywordIndex import KeywordIndex
from Products.ZCTextIndex.OkapiIndex import OkapiIndex
from Products.ZCTextIndex.ZCTextIndex import PLexicon
from Products.ZCTextIndex.ZCTextIndex import ZCTextIndex
from ZODB.DB import DB
from ZODB.DemoStorage import DemoStorage
import transaction
def createDatabase():
# Create a DemoStorage and put an Application in it
db = DB(DemoStorage())
conn = db.open()
root = conn.root()
app = OFS.Application.Application()
root['Application'] = app
transaction.commit()
return app
app = createDatabase()
def sort(iterable, reverse=False):
L = list(iterable)
if reverse:
L.sort(reverse=True)
else:
L.sort()
return L
class zdummy(ExtensionClass.Base):
def __init__(self, num):
self.num = num
def title(self):
return '%d' % self.num
class dummy(ExtensionClass.Base):
att1 = 'att1'
att2 = 'att2'
att3 = ['att3']
def __init__(self, num):
self.num = num
def col1(self):
return 'col1'
def col2(self):
return 'col2'
def col3(self):
return ['col3']
class objRS(ExtensionClass.Base):
def __init__(self, num):
self.number = num
class CatalogBase(object):
def _makeOne(self):
from Products.ZCatalog.Catalog import Catalog
return Catalog()
def setUp(self):
self._catalog = self._makeOne()
def tearDown(self):
self._catalog = None
class TestAddDelColumn(CatalogBase, unittest.TestCase):
def testAdd(self):
self._catalog.addColumn('id')
self.assertEqual('id' in self._catalog.schema, True,
'add column failed')
def testAddBad(self):
from Products.ZCatalog.Catalog import CatalogError
self.assertRaises(CatalogError, self._catalog.addColumn, '_id')
def testDel(self):
self._catalog.addColumn('id')
self._catalog.delColumn('id')
self.assert_('id' not in self._catalog.schema,
'del column failed')
class TestAddDelIndexes(CatalogBase, unittest.TestCase):
def testAddFieldIndex(self):
idx = FieldIndex('id')
self._catalog.addIndex('id', idx)
self.assert_(isinstance(self._catalog.indexes['id'],
type(FieldIndex('id'))),
'add field index failed')
def testAddTextIndex(self):
self._catalog.lexicon = PLexicon('lexicon')
idx = ZCTextIndex('id', caller=self._catalog,
index_factory=OkapiIndex, lexicon_id='lexicon')
self._catalog.addIndex('id', idx)
i = self._catalog.indexes['id']
self.assert_(isinstance(i, ZCTextIndex), 'add text index failed')
def testAddKeywordIndex(self):
idx = KeywordIndex('id')
self._catalog.addIndex('id', idx)
i = self._catalog.indexes['id']
self.assert_(isinstance(i, type(KeywordIndex('id'))),
'add kw index failed')
def testDelFieldIndex(self):
idx = FieldIndex('id')
self._catalog.addIndex('id', idx)
self._catalog.delIndex('id')
self.assert_('id' not in self._catalog.indexes,
'del index failed')
def testDelTextIndex(self):
self._catalog.lexicon = PLexicon('lexicon')
idx = ZCTextIndex('id', caller=self._catalog,
index_factory=OkapiIndex, lexicon_id='lexicon')
self._catalog.addIndex('id', idx)
self._catalog.delIndex('id')
self.assert_('id' not in self._catalog.indexes,
'del index failed')
def testDelKeywordIndex(self):
idx = KeywordIndex('id')
self._catalog.addIndex('id', idx)
self._catalog.delIndex('id')
self.assert_('id' not in self._catalog.indexes,
'del index failed')
class TestCatalog(CatalogBase, unittest.TestCase):
upper = 100
nums = range(upper)
for i in range(upper):
j = random.randrange(0, upper)
tmp = nums[i]
nums[i] = nums[j]
nums[j] = tmp
def setUp(self):
self._catalog = self._makeOne()
self._catalog.lexicon = PLexicon('lexicon')
col1 = FieldIndex('col1')
col2 = ZCTextIndex('col2', caller=self._catalog,
index_factory=OkapiIndex, lexicon_id='lexicon')
col3 = KeywordIndex('col3')
self._catalog.addIndex('col1', col1)
self._catalog.addIndex('col2', col2)
self._catalog.addIndex('col3', col3)
self._catalog.addColumn('col1')
self._catalog.addColumn('col2')
self._catalog.addColumn('col3')
att1 = FieldIndex('att1')
att2 = ZCTextIndex('att2', caller=self._catalog,
index_factory=OkapiIndex, lexicon_id='lexicon')
att3 = KeywordIndex('att3')
num = FieldIndex('num')
self._catalog.addIndex('att1', att1)
self._catalog.addIndex('att2', att2)
self._catalog.addIndex('att3', att3)
self._catalog.addIndex('num', num)
self._catalog.addColumn('att1')
self._catalog.addColumn('att2')
self._catalog.addColumn('att3')
self._catalog.addColumn('num')
for x in range(0, self.upper):
self._catalog.catalogObject(dummy(self.nums[x]), repr(x))
self._catalog = self._catalog.__of__(dummy('foo'))
# clear
# updateBrains
# __getitem__
# __setstate__
# useBrains
# getIndex
# updateMetadata
def testCatalogObjectUpdateMetadataFalse(self):
ob = dummy(9999)
self._catalog.catalogObject(ob, `9999`)
brain = self._catalog(num=9999)[0]
self.assertEqual(brain.att1, 'att1')
ob.att1 = 'foobar'
self._catalog.catalogObject(ob, `9999`, update_metadata=0)
brain = self._catalog(num=9999)[0]
self.assertEqual(brain.att1, 'att1')
self._catalog.catalogObject(ob, `9999`)
brain = self._catalog(num=9999)[0]
self.assertEqual(brain.att1, 'foobar')
def uncatalog(self):
for x in range(0, self.upper):
self._catalog.uncatalogObject(`x`)
def testUncatalogFieldIndex(self):
self.uncatalog()
a = self._catalog(att1='att1')
self.assertEqual(len(a), 0, 'len: %s' % len(a))
def testUncatalogTextIndex(self):
self.uncatalog()
a = self._catalog(att2='att2')
self.assertEqual(len(a), 0, 'len: %s' % len(a))
def testUncatalogKeywordIndex(self):
self.uncatalog()
a = self._catalog(att3='att3')
self.assertEqual(len(a), 0, 'len: %s' % len(a))
def testBadUncatalog(self):
try:
self._catalog.uncatalogObject('asdasdasd')
except Exception:
self.fail('uncatalogObject raised exception on bad uid')
def testUncatalogTwice(self):
self._catalog.uncatalogObject(`0`)
def _second(self):
self._catalog.uncatalogObject(`0`)
self.assertRaises(Exception, _second)
def testCatalogLength(self):
for x in range(0, self.upper):
self._catalog.uncatalogObject(`x`)
self.assertEqual(len(self._catalog), 0)
def testUniqueValuesForLength(self):
a = self._catalog.uniqueValuesFor('att1')
self.assertEqual(len(a), 1, 'bad number of unique values %s' % a)
def testUniqueValuesForContent(self):
a = self._catalog.uniqueValuesFor('att1')
self.assertEqual(a[0], 'att1', 'bad content %s' % a[0])
# hasuid
# recordify
# instantiate
# getMetadataForRID
# getIndexDataForRID
# make_query
def test_sorted_search_indexes_empty(self):
result = self._catalog._sorted_search_indexes({})
self.assertEquals(len(result), 0)
def test_sorted_search_indexes_one(self):
result = self._catalog._sorted_search_indexes({'att1': 'a'})
self.assertEquals(result, ['att1'])
def test_sorted_search_indexes_many(self):
query = {'att1': 'a', 'att2': 'b', 'num': 1}
result = self._catalog._sorted_search_indexes(query)
self.assertEquals(set(result), set(['att1', 'att2', 'num']))
def test_sorted_search_indexes_priority(self):
# att2 and col2 don't support ILimitedResultIndex, att1 does
query = {'att1': 'a', 'att2': 'b', 'col2': 'c'}
result = self._catalog._sorted_search_indexes(query)
self.assertEquals(result.index('att1'), 2)
# search
# sortResults
# _get_sort_attr
# _getSortIndex
# searchResults
def testResultLength(self):
a = self._catalog(att1='att1')
self.assertEqual(len(a), self.upper,
'length should be %s, its %s' % (self.upper, len(a)))
def testMappingWithEmptyKeysDoesntReturnAll(self):
# Queries with empty keys used to return all, because of a bug in the
# parseIndexRequest function, mistaking a CatalogSearchArgumentsMap
# for a Record class
a = self._catalog({'col1': '', 'col2': '', 'col3': ''})
self.assertEqual(len(a), 0, 'length should be 0, its %s' % len(a))
def testFieldIndexLength(self):
a = self._catalog(att1='att1')
self.assertEqual(len(a), self.upper,
'should be %s, but is %s' % (self.upper, len(a)))
def testTextIndexLength(self):
a = self._catalog(att2='att2')
self.assertEqual(len(a), self.upper,
'should be %s, but is %s' % (self.upper, len(a)))
def testKeywordIndexLength(self):
a = self._catalog(att3='att3')
self.assertEqual(len(a), self.upper,
'should be %s, but is %s' % (self.upper, len(a)))
def testGoodSortIndex(self):
upper = self.upper
a = self._catalog(att1='att1', sort_on='num')
self.assertEqual(len(a), upper,
'length should be %s, its %s' % (upper, len(a)))
for x in range(self.upper):
self.assertEqual(a[x].num, x)
def testBadSortIndex(self):
from Products.ZCatalog.Catalog import CatalogError
def badsortindex():
self._catalog(sort_on='foofaraw')
self.assertRaises(CatalogError, badsortindex)
def testWrongKindOfIndexForSort(self):
from Products.ZCatalog.Catalog import CatalogError
def wrongsortindex():
self._catalog(sort_on='att2')
self.assertRaises(CatalogError, wrongsortindex)
def testTextIndexQWithSortOn(self):
upper = self.upper
a = self._catalog(sort_on='num', att2='att2')
self.assertEqual(len(a), upper,
'length should be %s, its %s' % (upper, len(a)))
for x in range(self.upper):
self.assertEqual(a[x].num, x)
def testTextIndexQWithoutSortOn(self):
upper = self.upper
a = self._catalog(att2='att2')
self.assertEqual(len(a), upper,
'length should be %s, its %s' % (upper, len(a)))
def testKeywordIndexWithMinRange(self):
a = self._catalog(att3={'query': 'att', 'range': 'min'})
self.assertEqual(len(a), self.upper)
def testKeywordIndexWithMaxRange(self):
a = self._catalog(att3={'query': 'att35', 'range': ':max'})
self.assertEqual(len(a), self.upper)
def testKeywordIndexWithMinMaxRangeCorrectSyntax(self):
a = self._catalog(att3={'query': ['att', 'att35'], 'range': 'min:max'})
self.assertEqual(len(a), self.upper)
def testKeywordIndexWithMinMaxRangeWrongSyntax(self):
# checkKeywordIndex with min/max range wrong syntax.
a = self._catalog(att3={'query': ['att'], 'range': 'min:max'})
self.assert_(len(a) != self.upper)
def testCombinedTextandKeywordQuery(self):
a = self._catalog(att3='att3', att2='att2')
self.assertEqual(len(a), self.upper)
def testLargeSortedResultSetWithSmallIndex(self):
# This exercises the optimization in the catalog that iterates
# over the sort index rather than the result set when the result
# set is much larger than the sort index.
a = self._catalog(att1='att1', sort_on='att1')
self.assertEqual(len(a), self.upper)
def testBadSortLimits(self):
self.assertRaises(AssertionError,
self._catalog, att1='att1', sort_on='num', sort_limit=0)
self.assertRaises(AssertionError,
self._catalog, att1='att1', sort_on='num', sort_limit=-10)
def testSortLimit(self):
full = self._catalog(att1='att1', sort_on='num')
a = self._catalog(att1='att1', sort_on='num', sort_limit=10)
self.assertEqual([r.num for r in a], [r.num for r in full[:10]])
self.assertEqual(a.actual_result_count, self.upper)
a = self._catalog(att1='att1', sort_on='num',
sort_limit=10, sort_order='reverse')
rev = [r.num for r in full[-10:]]
rev.reverse()
self.assertEqual([r.num for r in a], rev)
self.assertEqual(a.actual_result_count, self.upper)
def testBigSortLimit(self):
a = self._catalog(att1='att1', sort_on='num', sort_limit=self.upper*3)
self.assertEqual(a.actual_result_count, self.upper)
self.assertEqual(a[0].num, 0)
a = self._catalog(att1='att1',
sort_on='num', sort_limit=self.upper*3, sort_order='reverse')
self.assertEqual(a.actual_result_count, self.upper)
self.assertEqual(a[0].num, self.upper - 1)
class TestRangeSearch(CatalogBase, unittest.TestCase):
def setUp(self):
self._catalog = self._makeOne()
index = FieldIndex('number')
self._catalog.addIndex('number', index)
self._catalog.addColumn('number')
for i in range(5000):
obj = objRS(random.randrange(0, 20000))
self._catalog.catalogObject(obj, i)
self._catalog = self._catalog.__of__(objRS(200))
def testRangeSearch(self):
for i in range(1000):
m = random.randrange(0, 20000)
n = m + 1000
for r in self._catalog.searchResults(
number={'query': (m, n), 'range': 'min:max'}):
size = r.number
self.assert_(m<=size and size<=n,
"%d vs [%d,%d]" % (r.number, m, n))
class TestCatalogReturnAll(CatalogBase, unittest.TestCase):
def setUp(self):
self.warningshook = WarningsHook()
self.warningshook.install()
self._catalog = self._makeOne()
def testEmptyMappingReturnsAll(self):
col1 = FieldIndex('col1')
self._catalog.addIndex('col1', col1)
for x in range(0, 10):
self._catalog.catalogObject(dummy(x), repr(x))
self.assertEqual(len(self._catalog), 10)
length = len(self._catalog({}))
self.assertEqual(length, 10)
def tearDown(self):
CatalogBase.tearDown(self)
self.warningshook.uninstall()
class TestCatalogSearchArgumentsMap(unittest.TestCase):
def _makeOne(self, request=None, keywords=None):
from Products.ZCatalog.Catalog import CatalogSearchArgumentsMap
return CatalogSearchArgumentsMap(request, keywords)
def test_init_empty(self):
argmap = self._makeOne()
self.assert_(argmap)
def test_init_request(self):
argmap = self._makeOne(dict(foo='bar'), None)
self.assertEquals(argmap.get('foo'), 'bar')
def test_init_keywords(self):
argmap = self._makeOne(None, dict(foo='bar'))
self.assertEquals(argmap.get('foo'), 'bar')
def test_getitem(self):
argmap = self._makeOne(dict(a='a'), dict(b='b'))
self.assertEquals(argmap['a'], 'a')
self.assertEquals(argmap['b'], 'b')
self.assertRaises(KeyError, argmap.__getitem__, 'c')
def test_getitem_emptystring(self):
argmap = self._makeOne(dict(a='', c='c'), dict(b='', c=''))
self.assertRaises(KeyError, argmap.__getitem__, 'a')
self.assertRaises(KeyError, argmap.__getitem__, 'b')
self.assertEquals(argmap['c'], 'c')
def test_get(self):
argmap = self._makeOne(dict(a='a'), dict(b='b'))
self.assertEquals(argmap.get('a'), 'a')
self.assertEquals(argmap.get('b'), 'b')
self.assertEquals(argmap.get('c'), None)
self.assertEquals(argmap.get('c', 'default'), 'default')
def test_keywords_precedence(self):
argmap = self._makeOne(dict(a='a', c='r'), dict(b='b', c='k'))
self.assertEquals(argmap.get('c'), 'k')
self.assertEquals(argmap['c'], 'k')
def test_haskey(self):
argmap = self._makeOne(dict(a='a'), dict(b='b'))
self.assert_(argmap.has_key('a'))
self.assert_(argmap.has_key('b'))
self.assert_(not argmap.has_key('c'))
def test_contains(self):
argmap = self._makeOne(dict(a='a'), dict(b='b'))
self.assert_('a' in argmap)
self.assert_('b' in argmap)
self.assert_('c' not in argmap)
class TestMergeResults(CatalogBase, unittest.TestCase):
def setUp(self):
self.catalogs = []
for i in range(3):
cat = self._makeOne()
cat.lexicon = PLexicon('lexicon')
cat.addIndex('num', FieldIndex('num'))
cat.addIndex('big', FieldIndex('big'))
cat.addIndex('number', FieldIndex('number'))
i = ZCTextIndex('title', caller=cat, index_factory=OkapiIndex,
lexicon_id='lexicon')
cat.addIndex('title', i)
cat = cat.__of__(zdummy(16336))
for i in range(10):
obj = zdummy(i)
obj.big = i > 5
obj.number = True
cat.catalogObject(obj, str(i))
self.catalogs.append(cat)
def testNoFilterOrSort(self):
from Products.ZCatalog.Catalog import mergeResults
results = [cat.searchResults(
dict(number=True), _merge=0) for cat in self.catalogs]
merged_rids = [r.getRID() for r in mergeResults(
results, has_sort_keys=False, reverse=False)]
expected = [r.getRID() for r in chain(*results)]
self.assertEqual(sort(merged_rids), sort(expected))
def testSortedOnly(self):
from Products.ZCatalog.Catalog import mergeResults
results = [cat.searchResults(
dict(number=True, sort_on='num'), _merge=0)
for cat in self.catalogs]
merged_rids = [r.getRID() for r in mergeResults(
results, has_sort_keys=True, reverse=False)]
expected = sort(chain(*results))
expected = [rid for sortkey, rid, getitem in expected]
self.assertEqual(merged_rids, expected)
def testSortReverse(self):
from Products.ZCatalog.Catalog import mergeResults
results = [cat.searchResults(
dict(number=True, sort_on='num'), _merge=0)
for cat in self.catalogs]
merged_rids = [r.getRID() for r in mergeResults(
results, has_sort_keys=True, reverse=True)]
expected = sort(chain(*results), reverse=True)
expected = [rid for sortkey, rid, getitem in expected]
self.assertEqual(merged_rids, expected)
def testLimitSort(self):
from Products.ZCatalog.Catalog import mergeResults
results = [cat.searchResults(
dict(att1='att1', number=True, sort_on='num',
sort_limit=2), _merge=0)
for cat in self.catalogs]
merged_rids = [r.getRID() for r in mergeResults(
results, has_sort_keys=True, reverse=False)]
expected = sort(chain(*results))
expected = [rid for sortkey, rid, getitem in expected]
self.assertEqual(merged_rids, expected)
def testScored(self):
from Products.ZCatalog.Catalog import mergeResults
results = [cat.searchResults(title='4 or 5 or 6', _merge=0)
for cat in self.catalogs]
merged_rids = [r.getRID() for r in mergeResults(
results, has_sort_keys=True, reverse=False)]
expected = sort(chain(*results))
expected = [rid for sortkey, (nscore, score, rid), getitem in expected]
self.assertEqual(merged_rids, expected)
def testSmallIndexSort(self):
# Test that small index sort optimization is not used for merging
from Products.ZCatalog.Catalog import mergeResults
results = [cat.searchResults(
dict(number=True, sort_on='big'), _merge=0)
for cat in self.catalogs]
merged_rids = [r.getRID() for r in mergeResults(
results, has_sort_keys=True, reverse=False)]
expected = sort(chain(*results))
expected = [rid for sortkey, rid, getitem in expected]
self.assertEqual(merged_rids, expected)
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestAddDelColumn))
suite.addTest(unittest.makeSuite(TestAddDelIndexes))
suite.addTest(unittest.makeSuite(TestCatalog))
suite.addTest(unittest.makeSuite(TestRangeSearch))
suite.addTest(unittest.makeSuite(TestCatalogReturnAll))
suite.addTest(unittest.makeSuite(TestCatalogSearchArgumentsMap))
suite.addTest(unittest.makeSuite(TestMergeResults))
return suite
src/Products/ZCatalog/tests/test_lazy.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Unittests for Lazy sequence classes
"""
import unittest
class BaseSequenceTest(unittest.TestCase):
def _compare(self, lseq, seq):
self.assertEqual(len(lseq), len(seq))
self.assertEqual(list(lseq), seq)
class TestLazyCat(BaseSequenceTest):
def _createLSeq(self, *sequences):
from Products.ZCatalog.Lazy import LazyCat
return LazyCat(sequences)
def test_empty(self):
lcat = self._createLSeq([])
self._compare(lcat, [])
def test_repr(self):
lcat = self._createLSeq([0, 1])
self.assertEquals(repr(lcat), repr([0, 1]))
def test_init_single(self):
seq = range(10)
lcat = self._createLSeq(seq)
self._compare(lcat, seq)
def test_add(self):
seq1 = range(10)
seq2 = range(10, 20)
lcat1 = self._createLSeq(seq1)
lcat2 = self._createLSeq(seq2)
lcat = lcat1 + lcat2
self._compare(lcat, range(20))
def test_init_multiple(self):
from string import hexdigits, letters
seq1 = range(10)
seq2 = list(hexdigits)
seq3 = list(letters)
lcat = self._createLSeq(seq1, seq2, seq3)
self._compare(lcat, seq1 + seq2 + seq3)
def test_init_nested(self):
from string import hexdigits, letters
seq1 = range(10)
seq2 = list(hexdigits)
seq3 = list(letters)
lcat = apply(self._createLSeq,
[self._createLSeq(seq) for seq in (seq1, seq2, seq3)])
self._compare(lcat, seq1 + seq2 + seq3)
def test_slicing(self):
from string import hexdigits, letters
seq1 = range(10)
seq2 = list(hexdigits)
seq3 = list(letters)
lcat = apply(self._createLSeq,
[self._createLSeq(seq) for seq in (seq1, seq2, seq3)])
self._compare(lcat[5:-5], seq1[5:] + seq2 + seq3[:-5])
def test_length(self):
# Unaccessed length
lcat = self._createLSeq(range(10))
self.assertEqual(len(lcat), 10)
# Accessed in the middle
lcat = self._createLSeq(range(10))
lcat[4]
self.assertEqual(len(lcat), 10)
# Accessed after the lcat is accessed over the whole range
lcat = self._createLSeq(range(10))
lcat[:]
self.assertEqual(len(lcat), 10)
class TestLazyMap(TestLazyCat):
def _createLSeq(self, *seq):
return self._createLMap(lambda x: x, *seq)
def _createLMap(self, mapfunc, *seq):
from Products.ZCatalog.Lazy import LazyMap
totalseq = []
for s in seq:
totalseq.extend(s)
return LazyMap(mapfunc, totalseq)
def test_map(self):
from string import hexdigits, letters
seq1 = range(10)
seq2 = list(hexdigits)
seq3 = list(letters)
filter = lambda x: str(x).lower()
lmap = self._createLMap(filter, seq1, seq2, seq3)
self._compare(lmap, [str(x).lower() for x in (seq1 + seq2 + seq3)])
def testMapFuncIsOnlyCalledAsNecessary(self):
seq = range(10)
count = [0] # closure only works with list, and `nonlocal` in py3
def func(x):
count[0] += 1
return x
lmap = self._createLMap(func, seq)
self.assertEqual(lmap[5], 5)
self.assertEqual(count[0], 1)
class TestLazyFilter(TestLazyCat):
def _createLSeq(self, *seq):
return self._createLFilter(lambda x: True, *seq)
def _createLFilter(self, filter, *seq):
from Products.ZCatalog.Lazy import LazyFilter
totalseq = []
for s in seq:
totalseq.extend(s)
return LazyFilter(filter, totalseq)
def test_filter(self):
from string import hexdigits, letters
seq1 = range(10)
seq2 = list(hexdigits)
seq3 = list(letters)
filter = lambda x: str(x).isalpha()
lmap = self._createLFilter(filter, seq1, seq2, seq3)
self._compare(lmap, seq2[10:] + seq3)
def test_length_with_filter(self):
from string import letters
# Unaccessed length
lfilter = self._createLFilter(lambda x: x.islower(), list(letters))
self.assertEqual(len(lfilter), 26)
# Accessed in the middle
lfilter = self._createLFilter(lambda x: x.islower(), list(letters))
lfilter[13]
self.assertEqual(len(lfilter), 26)
# Accessed after the lcat is accessed over the whole range
lfilter = self._createLFilter(lambda x: x.islower(), list(letters))
lfilter[:]
self.assertEqual(len(lfilter), 26)
class TestLazyMop(TestLazyCat):
def _createLSeq(self, *seq):
return self._createLMop(lambda x: x, *seq)
def _createLMop(self, mapfunc, *seq):
from Products.ZCatalog.Lazy import LazyMop
totalseq = []
for s in seq:
totalseq.extend(s)
return LazyMop(mapfunc, totalseq)
def test_mop(self):
from string import hexdigits, letters
seq1 = range(10)
seq2 = list(hexdigits)
seq3 = list(letters)
def filter(x):
if isinstance(x, int):
raise ValueError
return x.lower()
lmop = self._createLMop(filter, seq1, seq2, seq3)
self._compare(lmop, [str(x).lower() for x in (seq2 + seq3)])
def test_length_with_filter(self):
from string import letters
seq = range(10) + list(letters)
def filter(x):
if isinstance(x, int):
raise ValueError
return x.lower()
# Unaccessed length
lmop = self._createLMop(filter, seq)
self.assertEqual(len(lmop), 52)
# Accessed in the middle
lmop = self._createLMop(filter, seq)
lmop[26]
self.assertEqual(len(lmop), 52)
# Accessed after the lcat is accessed over the whole range
lmop = self._createLMop(filter, letters)
lmop[:]
self.assertEqual(len(lmop), 52)
class TestLazyValues(BaseSequenceTest):
def _createLValues(self, seq):
from Products.ZCatalog.Lazy import LazyValues
return LazyValues(seq)
def test_empty(self):
lvals = self._createLValues([])
self._compare(lvals, [])
def test_values(self):
from string import letters
seq = zip(letters, range(10))
lvals = self._createLValues(seq)
self._compare(lvals, range(10))
def test_slicing(self):
from string import letters
seq = zip(letters, range(10))
lvals = self._createLValues(seq)
self._compare(lvals[2:-2], range(2, 8))
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestLazyCat))
suite.addTest(unittest.makeSuite(TestLazyMap))
suite.addTest(unittest.makeSuite(TestLazyFilter))
suite.addTest(unittest.makeSuite(TestLazyMop))
suite.addTest(unittest.makeSuite(TestLazyValues))
return suite
src/Products/ZCatalog/tests/test_plan.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2010 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import os
import time
import unittest
from zope.testing import cleanup
class dummy(object):
def __init__(self, num):
self.num = num
def big(self):
return self.num > 5
def numbers(self):
return (self.num, self.num + 1)
TESTMAP = {
'/folder/catalog': {
'index1 index2': {
'index1': (10, 2.0, 3, True),
'index2': (15, 1.5, 2, False),
}
}
}
class TestNestedDict(unittest.TestCase):
def setUp(self):
self.nest = self._makeOne()
def _makeOne(self):
from ..plan import NestedDict
return NestedDict
def test_novalue(self):
self.assertEquals(getattr(self.nest, 'value', None), None)
def test_nolock(self):
self.assertEquals(getattr(self.nest, 'lock', None), None)
class TestPriorityMap(unittest.TestCase):
def setUp(self):
self.pmap = self._makeOne()
def tearDown(self):
self.pmap.clear()
def _makeOne(self):
from ..plan import PriorityMap
return PriorityMap
def test_get_value(self):
self.assertEquals(self.pmap.get_value(), {})
def test_get(self):
self.assertEquals(self.pmap.get('foo'), {})
def test_set(self):
self.pmap.set('foo', {'bar': 1})
self.assertEquals(self.pmap.get('foo'), {'bar': 1})
def test_clear(self):
self.pmap.set('foo', {'bar': 1})
self.pmap.clear()
self.assertEquals(self.pmap.value, {})
def test_get_entry(self):
self.assertEquals(self.pmap.get_entry('foo', 'bar'), {})
def test_set_entry(self):
self.pmap.set_entry('foo', 'bar', {'baz': 1})
self.assertEquals(self.pmap.get_entry('foo', 'bar'), {'baz': 1})
def test_clear_entry(self):
self.pmap.set('foo', {'bar': 1})
self.pmap.clear_entry('foo')
self.assertEquals(self.pmap.get('foo'), {})
class TestPriorityMapDefault(unittest.TestCase):
def setUp(self):
self.pmap = self._makeOne()
def tearDown(self):
self.pmap.clear()
def _makeOne(self):
from ..plan import PriorityMap
return PriorityMap
def test_empty(self):
self.pmap.load_default()
self.assertEquals(self.pmap.get_value(), {})
def test_load_failure(self):
try:
os.environ['ZCATALOGQUERYPLAN'] = 'Products.ZCatalog.invalid'
# 'Products.ZCatalog.tests.test_plan.TESTMAP'
self.pmap.load_default()
self.assertEquals(self.pmap.get_value(), {})
finally:
del os.environ['ZCATALOGQUERYPLAN']
def test_load(self):
from ..plan import Benchmark
try:
os.environ['ZCATALOGQUERYPLAN'] = \
'Products.ZCatalog.tests.test_plan.TESTMAP'
self.pmap.load_default()
expected = {'/folder/catalog': {'index1 index2': {
'index1': Benchmark(num=10, duration=2.0, hits=3, limit=True),
'index2': Benchmark(num=15, duration=1.5, hits=2, limit=False),
}}}
self.assertEquals(self.pmap.get_value(), expected)
finally:
del os.environ['ZCATALOGQUERYPLAN']
class TestReports(unittest.TestCase):
def setUp(self):
self.reports = self._makeOne()
def tearDown(self):
self.reports.clear()
def _makeOne(self):
from ..plan import Reports
return Reports
def test_value(self):
self.assertEquals(self.reports.value, {})
def test_lock(self):
from thread import LockType
self.assertEquals(type(self.reports.lock), LockType)
class TestValueIndexes(unittest.TestCase):
def setUp(self):
self.value = self._makeOne()
def tearDown(self):
self.value.clear()
def _makeOne(self):
from ..plan import ValueIndexes
return ValueIndexes
def test_get(self):
self.assertEquals(self.value.get(), frozenset())
def test_set(self):
indexes = ('index1', 'index2')
self.value.set(indexes)
self.assertEquals(self.value.get(), frozenset(indexes))
def test_clear(self):
self.value.set(('index1', ))
self.value.clear()
self.assertEquals(self.value.get(), frozenset())
def test_determine_already_set(self):
self.value.set(('index1', ))
self.assertEquals(self.value.determine(()), frozenset(('index1', )))
# class TestValueIndexesDetermination(unittest.TestCase):
# Test the actual logic for determining value indexes
# class TestMakeKey(unittest.TestCase):
class TestCatalogPlan(cleanup.CleanUp, unittest.TestCase):
def setUp(self):
cleanup.CleanUp.setUp(self)
from Products.ZCatalog.Catalog import Catalog
self.cat = Catalog('catalog')
def _makeOne(self, catalog=None, query=None):
from ..plan import CatalogPlan
if catalog is None:
catalog = self.cat
return CatalogPlan(catalog, query=query)
def test_get_id(self):
plan = self._makeOne()
self.assertEquals(plan.get_id(), ('', 'NonPersistentCatalog'))
def test_get_id_persistent(self):
from Products.ZCatalog.ZCatalog import ZCatalog
zcat = ZCatalog('catalog')
plan = self._makeOne(zcat._catalog)
self.assertEquals(plan.get_id(), ('catalog',))
def test_plan_empty(self):
plan = self._makeOne()
self.assertEquals(plan.plan(), None)
def test_start(self):
plan = self._makeOne()
plan.start()
self.assert_(plan.start_time <= time.time())
def test_start_split(self):
plan = self._makeOne()
plan.start_split('index1')
self.assert_('index1' in plan.interim)
def test_stop_split(self):
plan = self._makeOne()
plan.start_split('index1')
plan.stop_split('index1')
self.assert_('index1' in plan.interim)
i1 = plan.interim['index1']
self.assert_(i1.start <= i1.end)
self.assert_('index1' in plan.benchmark)
def test_stop_split_sort_on(self):
plan = self._makeOne()
plan.start_split('sort_on')
plan.stop_split('sort_on')
self.assert_('sort_on' in plan.interim)
so = plan.interim['sort_on']
self.assert_(so.start <= so.end)
self.assert_('sort_on' not in plan.benchmark)
def test_stop(self):
plan = self._makeOne(query={'index1': 1, 'index2': 2})
plan.start()
plan.start_split('index1')
plan.stop_split('index1')
plan.start_split('index1')
plan.stop_split('index1')
plan.start_split('sort_on')
plan.stop_split('sort_on')
time.sleep(0.02) # wait at least one Windows clock tick
plan.stop()
self.assert_(plan.duration > 0)
self.assert_('index1' in plan.benchmark)
self.assertEquals(plan.benchmark['index1'].hits, 2)
self.assert_('index2' in plan.benchmark)
self.assertEquals(plan.benchmark['index2'].hits, 0)
self.assertEquals(set(plan.plan()), set(('index1', 'index2')))
def test_log(self):
plan = self._makeOne(query={'index1': 1})
plan.threshold = 0.0
plan.start()
plan.start_split('index1')
plan.stop_split('index1')
plan.stop()
plan.log()
report = plan.report()
self.assertEquals(len(report), 1)
self.assertEquals(report[0]['counter'], 2)
plan.reset()
self.assertEquals(len(plan.report()), 0)
class TestCatalogReport(cleanup.CleanUp, unittest.TestCase):
def setUp(self):
cleanup.CleanUp.setUp(self)
from Products.ZCatalog.ZCatalog import ZCatalog
self.zcat = ZCatalog('catalog')
self.zcat.long_query_time = 0.0
self._add_indexes()
for i in range(9):
obj = dummy(i)
self.zcat.catalog_object(obj, str(i))
def _add_indexes(self):
from Products.PluginIndexes.FieldIndex.FieldIndex import FieldIndex
from Products.PluginIndexes.KeywordIndex.KeywordIndex import \
KeywordIndex
num = FieldIndex('num')
self.zcat._catalog.addIndex('num', num)
big = FieldIndex('big')
self.zcat._catalog.addIndex('big', big)
numbers = KeywordIndex('numbers')
self.zcat._catalog.addIndex('numbers', numbers)
def test_ReportLength(self):
""" tests the report aggregation """
self.zcat.manage_resetCatalogReport()
self.zcat.searchResults(numbers=4, sort_on='num')
self.zcat.searchResults(numbers=1, sort_on='num')
self.zcat.searchResults(numbers=3, sort_on='num')
self.zcat.searchResults(big=True, sort_on='num')
self.zcat.searchResults(big=True, sort_on='num')
self.zcat.searchResults(big=False, sort_on='num')
self.zcat.searchResults(num=[5, 4, 3], sort_on='num')
self.zcat.searchResults(num=(3, 4, 5), sort_on='num')
self.assertEqual(4, len(self.zcat.getCatalogReport()))
def test_ReportCounter(self):
""" tests the counter of equal queries """
self.zcat.manage_resetCatalogReport()
self.zcat.searchResults(numbers=5, sort_on='num')
self.zcat.searchResults(numbers=6, sort_on='num')
self.zcat.searchResults(numbers=8, sort_on='num')
r = self.zcat.getCatalogReport()[0]
self.assertEqual(r['counter'], 3)
def test_ReportKey(self):
""" tests the query keys for uniqueness """
# query key 1
key = ('sort_on', ('big', 'True'))
self.zcat.manage_resetCatalogReport()
self.zcat.searchResults(big=True, sort_on='num')
self.zcat.searchResults(big=True, sort_on='num')
r = self.zcat.getCatalogReport()[0]
self.assertEqual(r['query'], key)
self.assertEqual(r['counter'], 2)
# query key 2
key = ('sort_on', ('big', 'False'))
self.zcat.manage_resetCatalogReport()
self.zcat.searchResults(big=False, sort_on='num')
r = self.zcat.getCatalogReport()[0]
self.assertEqual(r['query'], key)
self.assertEqual(r['counter'], 1)
# query key 3
key = ('sort_on', ('num', '[3, 4, 5]'))
self.zcat.manage_resetCatalogReport()
self.zcat.searchResults(num=[5, 4, 3], sort_on='num')
self.zcat.searchResults(num=(3, 4, 5), sort_on='num')
r = self.zcat.getCatalogReport()[0]
self.assertEqual(r['query'], key)
self.assertEqual(r['counter'], 2)
src/Products/ZCatalog/tests/test_zcatalog.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
""" Unittests for ZCatalog
"""
import unittest
import Zope2
Zope2.startup()
from AccessControl.SecurityManagement import setSecurityManager
from AccessControl.SecurityManagement import noSecurityManager
from AccessControl import Unauthorized
from Acquisition import Implicit
import ExtensionClass
import OFS.Application
from OFS.Folder import Folder as OFS_Folder
from ZODB.DB import DB
from ZODB.DemoStorage import DemoStorage
import transaction
def createDatabase():
# Create a DemoStorage and put an Application in it
db = DB(DemoStorage())
conn = db.open()
root = conn.root()
app = OFS.Application.Application()
root['Application'] = app
transaction.commit()
return app
app = createDatabase()
class Folder(OFS_Folder):
def __init__(self, id):
self._setId(id)
OFS_Folder.__init__(self)
class zdummy(ExtensionClass.Base):
def __init__(self, num):
self.num = num
def title(self):
return '%d' % self.num
class zdummyFalse(zdummy):
def __nonzero__(self):
return False
class dummyLenFail(zdummy):
def __init__(self, num, fail):
zdummy.__init__(self, num)
self.fail = fail
def __len__(self):
self.fail("__len__() was called")
class dummyNonzeroFail(zdummy):
def __init__(self, num, fail):
zdummy.__init__(self, num)
self.fail = fail
def __nonzero__(self):
self.fail("__nonzero__() was called")
class FakeTraversalError(KeyError):
"""fake traversal exception for testing"""
class fakeparent(Implicit):
# fake parent mapping unrestrictedTraverse to
# catalog.resolve_path as simulated by TestZCatalog
marker = object()
def __init__(self, d):
self.d = d
def unrestrictedTraverse(self, path, default=marker):
result = self.d.get(path, default)
if result is self.marker:
raise FakeTraversalError(path)
return result
class PickySecurityManager:
def __init__(self, badnames=[]):
self.badnames = badnames
def validateValue(self, value):
return 1
def validate(self, accessed, container, name, value):
if name not in self.badnames:
return 1
raise Unauthorized(name)
class ZCatalogBase(object):
def _makeOne(self):
from Products.ZCatalog.ZCatalog import ZCatalog
return ZCatalog('Catalog')
def setUp(self):
self._catalog = self._makeOne()
def tearDown(self):
self._catalog = None
class TestZCatalog(ZCatalogBase, unittest.TestCase):
def setUp(self):
ZCatalogBase.setUp(self)
self._catalog.resolve_path = self._resolve_num
self._catalog.addIndex('title', 'KeywordIndex')
self._catalog.addColumn('title')
self.upper = 10
self.d = {}
for x in range(0, self.upper):
# make uid a string of the number
ob = zdummy(x)
self.d[str(x)] = ob
self._catalog.catalog_object(ob, str(x))
def _resolve_num(self, num):
return self.d[num]
def test_interfaces(self):
from Products.ZCatalog.interfaces import IZCatalog
from Products.ZCatalog.ZCatalog import ZCatalog
from zope.interface.verify import verifyClass
verifyClass(IZCatalog, ZCatalog)
def test_len(self):
self.assertEquals(len(self._catalog), self.upper)
# manage_edit
# manage_subbingToggle
def testBooleanEvalOn_manage_catalogObject(self):
self.d['11'] = dummyLenFail(11, self.fail)
self.d['12'] = dummyNonzeroFail(12, self.fail)
# create a fake response that doesn't bomb on manage_catalogObject()
class myresponse:
def redirect(self, url):
pass
# this next call should not fail
self._catalog.manage_catalogObject(None, myresponse(),
'URL1', urls=('11', '12'))
# manage_uncatalogObject
# manage_catalogReindex
def testBooleanEvalOn_refreshCatalog_getobject(self):
# wrap catalog under the fake parent providing unrestrictedTraverse()
catalog = self._catalog.__of__(fakeparent(self.d))
# replace entries to test refreshCatalog
self.d['0'] = dummyLenFail(0, self.fail)
self.d['1'] = dummyNonzeroFail(1, self.fail)
# this next call should not fail
catalog.refreshCatalog()
for uid in ('0', '1'):
rid = catalog.getrid(uid)
# neither should these
catalog.getobject(rid)
# manage_catalogClear
# manage_catalogFoundItems
# manage_addColumn
# manage_delColumn
# manage_addIndex
# manage_delIndex
# manage_clearIndex
def testReindexIndexDoesntDoMetadata(self):
self.d['0'].num = 9999
self._catalog.reindexIndex('title', {})
data = self._catalog.getMetadataForUID('0')
self.assertEqual(data['title'], '0')
def testReindexIndexesFalse(self):
# setup
false_id = self.upper + 1
ob = zdummyFalse(false_id)
self.d[str(false_id)] = ob
self._catalog.catalog_object(ob, str(false_id))
# test, object evaluates to false; there was bug which caused the
# object to be removed from index
ob.num = 9999
self._catalog.reindexIndex('title', {})
result = self._catalog(title='9999')
self.assertEquals(1, len(result))
# manage_reindexIndex
# catalog_object
# uncatalog_object
# uniqueValuesFor
# getpath
# getrid
def test_getobject_traversal(self):
# getobject doesn't mask TraversalErrors and doesn't delegate to
# resolve_url
# wrap catalog under the fake parent providing unrestrictedTraverse()
catalog = self._catalog.__of__(fakeparent(self.d))
# make resolve_url fail if ZCatalog falls back on it
def resolve_url(path, REQUEST):
self.fail(".resolve_url() should not be called by .getobject()")
catalog.resolve_url = resolve_url
# traversal should work at first
rid0 = catalog.getrid('0')
# lets set it up so the traversal fails
del self.d['0']
self.assertRaises(FakeTraversalError,
catalog.getobject, rid0, REQUEST=object())
# and if there is a None at the traversal point, that's where it
# should return
self.d['0'] = None
self.assertEquals(catalog.getobject(rid0), None)
def testGetMetadataForUID(self):
testNum = str(self.upper - 3) # as good as any..
data = self._catalog.getMetadataForUID(testNum)
self.assertEqual(data['title'], testNum)
def testGetIndexDataForUID(self):
testNum = str(self.upper - 3)
data = self._catalog.getIndexDataForUID(testNum)
self.assertEqual(data['title'][0], testNum)
def testUpdateMetadata(self):
self._catalog.catalog_object(zdummy(1), '1')
data = self._catalog.getMetadataForUID('1')
self.assertEqual(data['title'], '1')
self._catalog.catalog_object(zdummy(2), '1', update_metadata=0)
data = self._catalog.getMetadataForUID('1')
self.assertEqual(data['title'], '1')
self._catalog.catalog_object(zdummy(2), '1', update_metadata=1)
data = self._catalog.getMetadataForUID('1')
self.assertEqual(data['title'], '2')
# update_metadata defaults to true, test that here
self._catalog.catalog_object(zdummy(1), '1')
data = self._catalog.getMetadataForUID('1')
self.assertEqual(data['title'], '1')
# getMetadataForRID
# getIndexDataForRID
# schema
# indexes
# index_objects
# getIndexObjects
# _searchable_arguments
# _searchable_result_columns
def testSearchResults(self):
query = {'title': ['5', '6', '7']}
sr = self._catalog.searchResults(query)
self.assertEqual(len(sr), 3)
def testCall(self):
query = {'title': ['5', '6', '7']}
sr = self._catalog(query)
self.assertEqual(len(sr), 3)
def testSearch(self):
query = {'title': ['5', '6', '7']}
sr = self._catalog.search(query)
self.assertEqual(len(sr), 3)
# resolve_url
# resolve_path
# manage_normalize_paths
# manage_setProgress
# _getProgressThreshold
class TestAddDelColumnIndex(ZCatalogBase, unittest.TestCase):
def testAddIndex(self):
self._catalog.addIndex('id', 'FieldIndex')
self.assert_('id' in self._catalog.indexes())
def testDelIndex(self):
self._catalog.addIndex('title', 'FieldIndex')
self.assert_('title' in self._catalog.indexes())
self._catalog.delIndex('title')
self.assert_('title' not in self._catalog.indexes())
def testClearIndex(self):
self._catalog.addIndex('title', 'FieldIndex')
idx = self._catalog._catalog.getIndex('title')
for x in range(10):
ob = zdummy(x)
self._catalog.catalog_object(ob, str(x))
self.assertEquals(len(idx), 10)
self._catalog.clearIndex('title')
self.assertEquals(len(idx), 0)
def testAddColumn(self):
self._catalog.addColumn('num', default_value=0)
self.assert_('num' in self._catalog.schema())
def testDelColumn(self):
self._catalog.addColumn('title')
self._catalog.delColumn('title')
self.assert_('title' not in self._catalog.schema())
class TestZCatalogGetObject(ZCatalogBase, unittest.TestCase):
# Test what objects are returned by brain.getObject()
def setUp(self):
ZCatalogBase.setUp(self)
self._catalog.addIndex('id', 'FieldIndex')
root = Folder('')
root.getPhysicalRoot = lambda: root
self.root = root
self.root.catalog = self._catalog
def tearDown(self):
ZCatalogBase.tearDown(self)
noSecurityManager()
def test_getObject_found(self):
# Check normal traversal
root = self.root
catalog = root.catalog
root.ob = Folder('ob')
catalog.catalog_object(root.ob)
brain = catalog.searchResults({'id': 'ob'})[0]
self.assertEqual(brain.getPath(), '/ob')
self.assertEqual(brain.getObject().getId(), 'ob')
def test_getObject_missing_raises_NotFound(self):
# Check that if the object is missing we raise
from zExceptions import NotFound
root = self.root
catalog = root.catalog
root.ob = Folder('ob')
catalog.catalog_object(root.ob)
brain = catalog.searchResults({'id': 'ob'})[0]
del root.ob
self.assertRaises((NotFound, AttributeError, KeyError),
brain.getObject)
def test_getObject_restricted_raises_Unauthorized(self):
# Check that if the object's security does not allow traversal,
# None is returned
root = self.root
catalog = root.catalog
root.fold = Folder('fold')
root.fold.ob = Folder('ob')
catalog.catalog_object(root.fold.ob)
brain = catalog.searchResults({'id': 'ob'})[0]
# allow all accesses
pickySecurityManager = PickySecurityManager()
setSecurityManager(pickySecurityManager)
self.assertEqual(brain.getObject().getId(), 'ob')
# disallow just 'ob' access
pickySecurityManager = PickySecurityManager(['ob'])
setSecurityManager(pickySecurityManager)
self.assertRaises(Unauthorized, brain.getObject)
# disallow just 'fold' access
pickySecurityManager = PickySecurityManager(['fold'])
setSecurityManager(pickySecurityManager)
ob = brain.getObject()
self.assertFalse(ob is None)
self.assertEqual(ob.getId(), 'ob')
# Now test _unrestrictedGetObject
def test_unrestrictedGetObject_found(self):
# Check normal traversal
root = self.root
catalog = root.catalog
root.ob = Folder('ob')
catalog.catalog_object(root.ob)
brain = catalog.searchResults({'id': 'ob'})[0]
self.assertEqual(brain.getPath(), '/ob')
self.assertEqual(brain._unrestrictedGetObject().getId(), 'ob')
def test_unrestrictedGetObject_restricted(self):
# Check that if the object's security does not allow traversal,
# it's still is returned
root = self.root
catalog = root.catalog
root.fold = Folder('fold')
root.fold.ob = Folder('ob')
catalog.catalog_object(root.fold.ob)
brain = catalog.searchResults({'id': 'ob'})[0]
# allow all accesses
pickySecurityManager = PickySecurityManager()
setSecurityManager(pickySecurityManager)
self.assertEqual(brain._unrestrictedGetObject().getId(), 'ob')
# disallow just 'ob' access
pickySecurityManager = PickySecurityManager(['ob'])
setSecurityManager(pickySecurityManager)
self.assertEqual(brain._unrestrictedGetObject().getId(), 'ob')
# disallow just 'fold' access
pickySecurityManager = PickySecurityManager(['fold'])
setSecurityManager(pickySecurityManager)
self.assertEqual(brain._unrestrictedGetObject().getId(), 'ob')
def test_unrestrictedGetObject_missing_raises_NotFound(self):
# Check that if the object is missing we raise
from zExceptions import NotFound
root = self.root
catalog = root.catalog
root.ob = Folder('ob')
catalog.catalog_object(root.ob)
brain = catalog.searchResults({'id': 'ob'})[0]
del root.ob
self.assertRaises((NotFound, AttributeError, KeyError),
brain._unrestrictedGetObject)
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(TestZCatalog))
suite.addTest(unittest.makeSuite(TestAddDelColumnIndex))
suite.addTest(unittest.makeSuite(TestZCatalogGetObject))
return suite
versions.cfg
View file @ 97bb6b12
...@@ -22,6 +22,7 @@ Products.MIMETools = 2.13.0 ...@@ -22,6 +22,7 @@ Products.MIMETools = 2.13.0
Products.OFSP = 2.13.2 Products.OFSP = 2.13.2
Products.PythonScripts = 2.13.0 Products.PythonScripts = 2.13.0
Products.StandardCacheManagers = 2.13.0 Products.StandardCacheManagers = 2.13.0
Products.ZCatalog = 2.13.0
Products.ZCTextIndex = 2.13.1 Products.ZCTextIndex = 2.13.1
Record = 2.13.0 Record = 2.13.0
tempstorage = 2.12.1 tempstorage = 2.12.1
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7