Skip to content

Z
Zope
Commit 97bb6b12 authored by Hanno Schlichting's avatar Hanno Schlichting
Browse files
Options

Factored out the `Products.ZCatalog` and `Products.PluginIndexes` packages... 

Factored out the `Products.ZCatalog` and `Products.PluginIndexes` packages into a new `Products.ZCatalog` distribution.
parent 03a5df08
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 10 additions and 12182 deletions
buildout.cfg
View file @ 97bb6b12
...@@ -59,6 +59,7 @@ eggs = ...@@ -59,6 +59,7 @@ eggs =
Products.OFSP Products.OFSP
Products.PythonScripts Products.PythonScripts
Products.StandardCacheManagers Products.StandardCacheManagers
Products.ZCatalog
Products.ZCTextIndex Products.ZCTextIndex
Record Record
RestrictedPython RestrictedPython
......
doc/CHANGES.rst
View file @ 97bb6b12
...@@ -11,10 +11,14 @@ http://docs.zope.org/zope2/releases/. ...@@ -11,10 +11,14 @@ http://docs.zope.org/zope2/releases/.
Bugs Fixed Bugs Fixed
++++++++++ ++++++++++
- Fix `LazyMap` to avoid unnecessary function calls.
- LP 686664: WebDAV Lock Manager ZMI view wasn't accessible. - LP 686664: WebDAV Lock Manager ZMI view wasn't accessible.
Restructuring
+++++++++++++
- Factored out the `Products.ZCatalog` and `Products.PluginIndexes` packages
into a new `Products.ZCatalog` distribution.
2.13.1 (2010-12-07) 2.13.1 (2010-12-07)
------------------- -------------------
......
setup.py
View file @ 97bb6b12
...@@ -54,6 +54,7 @@ setup(name='Zope2', ...@@ -54,6 +54,7 @@ setup(name='Zope2',
'MultiMapping', 'MultiMapping',
'Persistence', 'Persistence',
'Products.OFSP >= 2.13.2', 'Products.OFSP >= 2.13.2',
'Products.ZCatalog',
'Products.ZCTextIndex', 'Products.ZCTextIndex',
'Record', 'Record',
'RestrictedPython', 'RestrictedPython',
......
sources.cfg
View file @ 97bb6b12
...@@ -15,6 +15,7 @@ Products.MIMETools = svn ^/Products.MIMETools/trunk ...@@ -15,6 +15,7 @@ Products.MIMETools = svn ^/Products.MIMETools/trunk
Products.OFSP = svn ^/Products.OFSP/trunk Products.OFSP = svn ^/Products.OFSP/trunk
Products.PythonScripts = svn ^/Products.PythonScripts/trunk Products.PythonScripts = svn ^/Products.PythonScripts/trunk
Products.StandardCacheManagers = svn ^/Products.StandardCacheManagers/trunk Products.StandardCacheManagers = svn ^/Products.StandardCacheManagers/trunk
Products.ZCatalog = svn ^/Products.ZCatalog/trunk
Products.ZCTextIndex = svn ^/Products.ZCTextIndex/trunk Products.ZCTextIndex = svn ^/Products.ZCTextIndex/trunk
Record = svn ^/Record/trunk Record = svn ^/Record/trunk
tempstorage = svn ^/tempstorage/trunk tempstorage = svn ^/tempstorage/trunk
......
src/Products/PluginIndexes/BooleanIndex/BooleanIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
from logging import getLogger
from App.special_dtml import DTMLFile
from BTrees.IIBTree import IIBTree, IITreeSet, IISet
from BTrees.IIBTree import union, intersection, difference
import BTrees.Length
from ZODB.POSException import ConflictError
from Products.PluginIndexes.common.util import parseIndexRequest
from Products.PluginIndexes.common.UnIndex import UnIndex
_marker = object()
LOG = getLogger('BooleanIndex.UnIndex')
class BooleanIndex(UnIndex):
"""Index for booleans
self._index = set([documentId1, documentId2])
self._unindex = {documentId:[True/False]}
False doesn't have actual entries in _index.
"""
meta_type = "BooleanIndex"
manage_options= (
{'label': 'Settings',
'action': 'manage_main'},
{'label': 'Browse',
'action': 'manage_browse'},
)
query_options = ["query"]
manage = manage_main = DTMLFile('dtml/manageBooleanIndex', globals())
manage_main._setName('manage_main')
manage_browse = DTMLFile('../dtml/browseIndex', globals())
def clear(self):
self._length = BTrees.Length.Length()
self._index = IITreeSet()
self._unindex = IIBTree()
def insertForwardIndexEntry(self, entry, documentId):
"""If True, insert directly into treeset
"""
if entry:
self._index.insert(documentId)
self._length.change(1)
def removeForwardIndexEntry(self, entry, documentId):
"""Take the entry provided and remove any reference to documentId
in its entry in the index.
"""
try:
if entry:
self._index.remove(documentId)
self._length.change(-1)
except ConflictError:
raise
except Exception:
LOG.exception('%s: unindex_object could not remove '
'documentId %s from index %s. This '
'should not happen.' % (self.__class__.__name__,
str(documentId), str(self.id)))
def _index_object(self, documentId, obj, threshold=None, attr=''):
""" index and object 'obj' with integer id 'documentId'"""
returnStatus = 0
# First we need to see if there's anything interesting to look at
datum = self._get_object_datum(obj, attr)
# Make it boolean, int as an optimization
datum = int(bool(datum))
# We don't want to do anything that we don't have to here, so we'll
# check to see if the new and existing information is the same.
oldDatum = self._unindex.get(documentId, _marker)
if datum != oldDatum:
if oldDatum is not _marker:
self.removeForwardIndexEntry(oldDatum, documentId)
if datum is _marker:
try:
del self._unindex[documentId]
except ConflictError:
raise
except Exception:
LOG.error('Should not happen: oldDatum was there, now '
'its not, for document with id %s' %
documentId)
if datum is not _marker:
if datum:
self.insertForwardIndexEntry(datum, documentId)
self._unindex[documentId] = datum
returnStatus = 1
return returnStatus
def _apply_index(self, request, resultset=None):
record = parseIndexRequest(request, self.id, self.query_options)
if record.keys is None:
return None
index = self._index
for key in record.keys:
if key:
# If True, check index
return (intersection(index, resultset), (self.id, ))
else:
# Otherwise, remove from resultset or _unindex
if resultset is None:
return (union(difference(self._unindex, index), IISet([])),
(self.id, ))
else:
return (difference(resultset, index), (self.id, ))
return (IISet(), (self.id, ))
def indexSize(self):
"""Return distinct values, as an optimization we always claim 2."""
return 2
def items(self):
items = []
for v, k in self._unindex.items():
if isinstance(v, int):
v = IISet((v, ))
items.append((k, v))
return items
manage_addBooleanIndexForm = DTMLFile('dtml/addBooleanIndex', globals())
def manage_addBooleanIndex(self, id, extra=None,
REQUEST=None, RESPONSE=None, URL3=None):
"""Add a boolean index"""
return self.manage_addIndex(id, 'BooleanIndex', extra=extra, \
REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)
src/Products/PluginIndexes/BooleanIndex/dtml/addBooleanIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var "manage_form_title(this(), _, form_title='Add BooleanIndex')">
<p class="form-help">
<strong>Boolean Indexes</strong> can be used for keeping track of
whether objects fulfills a certain contract, like isFolderish
</p>
<form action="manage_addBooleanIndex" method="post" enctype="multipart/form-data">
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Id
</div>
</td>
<td align="left" valign="top">
<input type="text" name="id" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-label">
Indexed attributes
</div>
</td>
<td align="left" valign="top">
<input type="text" name="extra.indexed_attrs:record:string" size="40" />
<em>attribute1,attribute2,...</em> or leave empty
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-optional">
Type
</div>
</td>
<td align="left" valign="top">
Boolean Index
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/BooleanIndex/dtml/manageBooleanIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p class="form-help">
Objects indexed: <dtml-var numObjects>
<br>
Distinct values: <dtml-var indexSize>
</p>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/BooleanIndex/tests.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2010 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import unittest
from BTrees.IIBTree import IISet
class Dummy(object):
def __init__(self, docid, truth):
self.id = docid
self.truth = truth
class TestBooleanIndex(unittest.TestCase):
def _getTargetClass(self):
from Products.PluginIndexes.BooleanIndex import BooleanIndex
return BooleanIndex.BooleanIndex
def _makeOne(self, attr='truth'):
return self._getTargetClass()(attr)
def test_index_true(self):
index = self._makeOne()
obj = Dummy(1, True)
index._index_object(obj.id, obj, attr='truth')
self.failUnless(1 in index._unindex)
self.failUnless(1 in index._index)
def test_index_false(self):
index = self._makeOne()
obj = Dummy(1, False)
index._index_object(obj.id, obj, attr='truth')
self.failUnless(1 in index._unindex)
self.failIf(1 in index._index)
def test_search_true(self):
index = self._makeOne()
obj = Dummy(1, True)
index._index_object(obj.id, obj, attr='truth')
obj = Dummy(2, False)
index._index_object(obj.id, obj, attr='truth')
res, idx = index._apply_index({'truth': True})
self.failUnlessEqual(idx, ('truth', ))
self.failUnlessEqual(list(res), [1])
def test_search_false(self):
index = self._makeOne()
obj = Dummy(1, True)
index._index_object(obj.id, obj, attr='truth')
obj = Dummy(2, False)
index._index_object(obj.id, obj, attr='truth')
res, idx = index._apply_index({'truth': False})
self.failUnlessEqual(idx, ('truth', ))
self.failUnlessEqual(list(res), [2])
def test_search_inputresult(self):
index = self._makeOne()
obj = Dummy(1, True)
index._index_object(obj.id, obj, attr='truth')
obj = Dummy(2, False)
index._index_object(obj.id, obj, attr='truth')
res, idx = index._apply_index({'truth': True}, resultset=IISet([]))
self.failUnlessEqual(idx, ('truth', ))
self.failUnlessEqual(list(res), [])
res, idx = index._apply_index({'truth': True}, resultset=IISet([2]))
self.failUnlessEqual(idx, ('truth', ))
self.failUnlessEqual(list(res), [])
res, idx = index._apply_index({'truth': True}, resultset=IISet([1]))
self.failUnlessEqual(idx, ('truth', ))
self.failUnlessEqual(list(res), [1])
res, idx = index._apply_index({'truth': True}, resultset=IISet([1, 2]))
self.failUnlessEqual(idx, ('truth', ))
self.failUnlessEqual(list(res), [1])
res, idx = index._apply_index({'truth': False},
resultset=IISet([1, 2]))
self.failUnlessEqual(idx, ('truth', ))
self.failUnlessEqual(list(res), [2])
def test_suite():
from unittest import TestSuite, makeSuite
suite = TestSuite()
suite.addTest(makeSuite(TestBooleanIndex))
return suite
src/Products/PluginIndexes/DateIndex/DateIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Date index.
"""
import time
from logging import getLogger
from datetime import date, datetime
from datetime import tzinfo, timedelta
from App.special_dtml import DTMLFile
from BTrees.IIBTree import IISet
from BTrees.IIBTree import union
from BTrees.IIBTree import intersection
from BTrees.IIBTree import multiunion
from BTrees.IOBTree import IOBTree
from BTrees.Length import Length
from BTrees.OIBTree import OIBTree
from DateTime.DateTime import DateTime
from OFS.PropertyManager import PropertyManager
from ZODB.POSException import ConflictError
from zope.interface import implements
from Products.PluginIndexes.common import safe_callable
from Products.PluginIndexes.common.UnIndex import UnIndex
from Products.PluginIndexes.common.util import parseIndexRequest
from Products.PluginIndexes.interfaces import IDateIndex
LOG = getLogger('DateIndex')
_marker = []
###############################################################################
# copied from Python 2.3 datetime.tzinfo docs
# A class capturing the platform's idea of local time.
ZERO = timedelta(0)
STDOFFSET = timedelta(seconds = -time.timezone)
if time.daylight:
DSTOFFSET = timedelta(seconds = -time.altzone)
else:
DSTOFFSET = STDOFFSET
DSTDIFF = DSTOFFSET - STDOFFSET
MAX32 = int(2**31 - 1)
class LocalTimezone(tzinfo):
def utcoffset(self, dt):
if self._isdst(dt):
return DSTOFFSET
else:
return STDOFFSET
def dst(self, dt):
if self._isdst(dt):
return DSTDIFF
else:
return ZERO
def tzname(self, dt):
return time.tzname[self._isdst(dt)]
def _isdst(self, dt):
tt = (dt.year, dt.month, dt.day,
dt.hour, dt.minute, dt.second,
dt.weekday(), 0, -1)
stamp = time.mktime(tt)
tt = time.localtime(stamp)
return tt.tm_isdst > 0
Local = LocalTimezone()
###############################################################################
class DateIndex(UnIndex, PropertyManager):
"""Index for dates.
"""
implements(IDateIndex)
meta_type = 'DateIndex'
query_options = ('query', 'range')
index_naive_time_as_local = True # False means index as UTC
_properties=({'id':'index_naive_time_as_local',
'type':'boolean',
'mode':'w'},)
manage = manage_main = DTMLFile( 'dtml/manageDateIndex', globals() )
manage_browse = DTMLFile('../dtml/browseIndex', globals())
manage_main._setName( 'manage_main' )
manage_options = ( { 'label' : 'Settings'
, 'action' : 'manage_main'
},
{'label': 'Browse',
'action': 'manage_browse',
},
) + PropertyManager.manage_options
def clear( self ):
""" Complete reset """
self._index = IOBTree()
self._unindex = OIBTree()
self._length = Length()
def index_object( self, documentId, obj, threshold=None ):
"""index an object, normalizing the indexed value to an integer
o Normalized value has granularity of one minute.
o Objects which have 'None' as indexed value are *omitted*,
by design.
"""
returnStatus = 0
try:
date_attr = getattr( obj, self.id )
if safe_callable( date_attr ):
date_attr = date_attr()
ConvertedDate = self._convert( value=date_attr, default=_marker )
except AttributeError:
ConvertedDate = _marker
oldConvertedDate = self._unindex.get( documentId, _marker )
if ConvertedDate != oldConvertedDate:
if oldConvertedDate is not _marker:
self.removeForwardIndexEntry(oldConvertedDate, documentId)
if ConvertedDate is _marker:
try:
del self._unindex[documentId]
except ConflictError:
raise
except:
LOG.error("Should not happen: ConvertedDate was there,"
" now it's not, for document with id %s" %
documentId)
if ConvertedDate is not _marker:
self.insertForwardIndexEntry( ConvertedDate, documentId )
self._unindex[documentId] = ConvertedDate
returnStatus = 1
return returnStatus
def _apply_index(self, request, resultset=None):
"""Apply the index to query parameters given in the argument
Normalize the 'query' arguments into integer values at minute
precision before querying.
"""
record = parseIndexRequest(request, self.id, self.query_options)
if record.keys is None:
return None
keys = map( self._convert, record.keys )
index = self._index
r = None
opr = None
#experimental code for specifing the operator
operator = record.get( 'operator', self.useOperator )
if not operator in self.operators :
raise RuntimeError("operator not valid: %s" % operator)
# depending on the operator we use intersection or union
if operator=="or":
set_func = union
else:
set_func = intersection
# range parameter
range_arg = record.get('range',None)
if range_arg:
opr = "range"
opr_args = []
if range_arg.find("min") > -1:
opr_args.append("min")
if range_arg.find("max") > -1:
opr_args.append("max")
if record.get('usage',None):
# see if any usage params are sent to field
opr = record.usage.lower().split(':')
opr, opr_args = opr[0], opr[1:]
if opr=="range": # range search
if 'min' in opr_args:
lo = min(keys)
else:
lo = None
if 'max' in opr_args:
hi = max(keys)
else:
hi = None
if hi:
setlist = index.values(lo,hi)
else:
setlist = index.values(lo)
r = multiunion(setlist)
else: # not a range search
for key in keys:
set = index.get(key, None)
if set is not None:
if isinstance(set, int):
set = IISet((set,))
else:
# set can't be bigger than resultset
set = intersection(set, resultset)
r = set_func(r, set)
if isinstance(r, int):
r = IISet((r,))
if r is None:
return IISet(), (self.id,)
else:
return r, (self.id,)
def _convert( self, value, default=None ):
"""Convert Date/Time value to our internal representation"""
# XXX: Code patched 20/May/2003 by Kiran Jonnalagadda to
# convert dates to UTC first.
if isinstance(value, DateTime):
t_tup = value.toZone('UTC').parts()
elif isinstance(value, (float, int)):
t_tup = time.gmtime( value )
elif isinstance(value, str) and value:
t_obj = DateTime( value ).toZone('UTC')
t_tup = t_obj.parts()
elif isinstance(value, datetime):
if self.index_naive_time_as_local and value.tzinfo is None:
value = value.replace(tzinfo=Local)
# else if tzinfo is None, naive time interpreted as UTC
t_tup = value.utctimetuple()
elif isinstance(value, date):
t_tup = value.timetuple()
else:
return default
yr = t_tup[0]
mo = t_tup[1]
dy = t_tup[2]
hr = t_tup[3]
mn = t_tup[4]
t_val = ( ( ( ( yr * 12 + mo ) * 31 + dy ) * 24 + hr ) * 60 + mn )
if t_val > MAX32:
# t_val must be integer fitting in the 32bit range
raise OverflowError(
"%s is not within the range of indexable dates (index: %s)"
% (value, self.id))
return t_val
manage_addDateIndexForm = DTMLFile( 'dtml/addDateIndex', globals() )
def manage_addDateIndex( self, id, REQUEST=None, RESPONSE=None, URL3=None):
"""Add a Date index"""
return self.manage_addIndex(id, 'DateIndex', extra=None, \
REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)
src/Products/PluginIndexes/DateIndex/README.txt deleted 100644 → 0
View file @ 03a5df08
DateIndex README
Overview
Normal FieldIndexes *can* be used to index values which are DateTime
instances, but they are hideously expensive:
o DateTime instances are *huge*, both in RAM and on disk.
o DateTime instances maintain an absurd amount of precision, far
beyond any reasonable search criteria for "normal" cases.
DateIndex is a pluggable index which addresses these two issues
as follows:
o It normalizes the indexed value to an integer representation
with a granularity of one minute.
o It normalizes the 'query' value into the same form.
o Objects which return 'None' for the index query are omitted from
the index.
src/Products/PluginIndexes/DateIndex/dtml/addDateIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var "manage_form_title(this(), _,
form_title='Add DateIndex',
)">
<p class="form-help">
A <em>DateIndex</em> indexes DateTime attributes.
</p>
<form action="manage_addDateIndex" method="post" enctype="multipart/form-data">
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Id
</div>
</td>
<td align="left" valign="top">
<input type="text" name="id" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-optional">
Type
</div>
</td>
<td align="left" valign="top">
DateIndex
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/DateIndex/dtml/manageDateIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p class="form-help">
Objects indexed: <dtml-var numObjects>
<br>
Distinct values: <dtml-var indexSize>
</p>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/DateIndex/tests/__init__.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
# This file is needed to make this a package.
src/Products/PluginIndexes/DateRangeIndex/README.txt deleted 100644 → 0
View file @ 03a5df08
DateRangeIndex README
Overview
Zope applications frequently wish to perform efficient queries
against a pair of date attributes/methods, representing a time
interval (e.g., effective / expiration dates). This query *can*
be done using a pair of indexes, but this implementation is
hideously expensive:
o DateTime instances are *huge*, both in RAM and on disk.
o DateTime instances maintain an absurd amount of precision, far
beyond any reasonable search criteria for "normal" cases.
o Results must be fetched and intersected between two indexes.
o Handling objects which do not specify both endpoints (i.e.,
where the interval is open or half-open) is iffy, as the
default value needs to be coerced into a different abnormal
value for each end to permit ordered comparison.
o The *very* common case of the open interval (neither endpoint
specified) should be optimized.
DateRangeIndex is a pluggable index which addresses these issues
as follows:
o It groups the "open" case into a special set, '_always'.
o It maintains separate ordered sets for each of the "half-open"
cases.
o It performs the expensive "intersect two range search" operation
only on the (usually small) set of objects which provide a
closed interval.
o It flattens the key values into integers with granularity of
one minute.
o It normalizes the 'query' value into the same form.
src/Products/PluginIndexes/DateRangeIndex/dtml/addDateRangeIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var "manage_form_title(this(), _,
form_title='Add DateRangeIndex')">
<p class="form-help">
A DateRangeIndex takes the name of two input attributes; one containing the
start date of the range, the second the end of the range. This index is filled
with range information based on those two markers. You can then search for
objects for those where a given date falls within the range.
</p>
<form action="manage_addDateRangeIndex" method="POST">
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Id
</div>
</td>
<td align="left" valign="top">
<input type="text" name="id" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-label">
Since field
</div>
</td>
<td align="left" valign="top">
<input type="text" name="extra.since_field:record" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-label">
Until field
</div>
</td>
<td align="left" valign="top">
<input type="text" name="extra.until_field:record" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/DateRangeIndex/dtml/manageDateRangeIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p class="form-help">
You can update this DateRangeIndex by editing the following field and clicking
<emUpdate</em>.
</p>
<p>
Objects indexed: <dtml-var numObjects>
<br>
Distinct values: <dtml-var indexSize>
</p>
<form action="&dtml-URL1;/manage_edit" method="POST">
<table cellpadding="2" cellspacing="0" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Since field
</td>
<td align="left" valign="top">
<input name="since_field" value="&dtml-getSinceField;" size="40" />
</td>
</tr>
<td align="left" valign="top">
<div class="form-label">
Until field
</td>
<td align="left" valign="top">
<input name="until_field" value="&dtml-getUntilField;" />
</td>
</tr>
<tr>
<td></td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value="Update">
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/DateRangeIndex/tests/__init__.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
# This file is needed to make this a package.
src/Products/PluginIndexes/DateRangeIndex/tests/test_DateRangeIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""DateRangeIndex unit tests.
"""
import unittest
class Dummy:
def __init__( self, name, start, stop ):
self._name = name
self._start = start
self._stop = stop
def name( self ):
return self._name
def start( self ):
return self._start
def stop( self ):
return self._stop
def datum( self ):
return ( self._start, self._stop )
dummies = [ Dummy( 'a', None, None )
, Dummy( 'b', None, None )
, Dummy( 'c', 0, None )
, Dummy( 'd', 10, None )
, Dummy( 'e', None, 4 )
, Dummy( 'f', None, 11 )
, Dummy( 'g', 0, 11 )
, Dummy( 'h', 2, 9 )
]
def matchingDummies( value ):
result = []
for dummy in dummies:
if ( ( dummy.start() is None or dummy.start() <= value )
and ( dummy.stop() is None or dummy.stop() >= value )
):
result.append( dummy )
return result
class DRI_Tests( unittest.TestCase ):
def _getTargetClass(self):
from Products.PluginIndexes.DateRangeIndex.DateRangeIndex \
import DateRangeIndex
return DateRangeIndex
def _makeOne(self,
id,
since_field=None,
until_field=None,
caller=None,
extra=None,
):
klass = self._getTargetClass()
return klass(id, since_field, until_field, caller, extra)
def test_interfaces(self):
from Products.PluginIndexes.interfaces import IDateRangeIndex
from Products.PluginIndexes.interfaces import IPluggableIndex
from Products.PluginIndexes.interfaces import ISortIndex
from Products.PluginIndexes.interfaces import IUniqueValueIndex
from zope.interface.verify import verifyClass
verifyClass(IDateRangeIndex, self._getTargetClass())
verifyClass(IPluggableIndex, self._getTargetClass())
verifyClass(ISortIndex, self._getTargetClass())
verifyClass(IUniqueValueIndex, self._getTargetClass())
def test_empty( self ):
empty = self._makeOne( 'empty' )
self.assertTrue(empty.getEntryForObject( 1234 ) is None)
empty.unindex_object( 1234 ) # shouldn't throw
self.assertFalse(empty.uniqueValues( 'foo' ))
self.assertFalse(empty.uniqueValues( 'foo', 1 ))
self.assertTrue(empty._apply_index( { 'zed' : 12345 } ) is None)
result, used = empty._apply_index( { 'empty' : 12345 } )
self.assertFalse(result)
self.assertEqual(used, ( None, None ))
def test_retrieval( self ):
index = self._makeOne( 'work', 'start', 'stop' )
for i in range( len( dummies ) ):
index.index_object( i, dummies[i] )
for i in range( len( dummies ) ):
self.assertEqual(index.getEntryForObject( i ), dummies[i].datum())
for value in range( -1, 15 ):
matches = matchingDummies( value )
results, used = index._apply_index( { 'work' : value } )
self.assertEqual(used, ( 'start', 'stop' ))
self.assertEqual(len( matches ), len( results ))
matches.sort( lambda x, y: cmp( x.name(), y.name() ) )
for result, match in map( None, results, matches ):
self.assertEqual(index.getEntryForObject(result), match.datum())
def test_longdates( self ):
self.assertRaises(OverflowError, self._badlong )
def _badlong(self):
import sys
index = self._makeOne ('work', 'start', 'stop' )
bad = Dummy( 'bad', long(sys.maxint) + 1, long(sys.maxint) + 1 )
index.index_object( 0, bad )
def test_datetime(self):
from datetime import datetime
from DateTime.DateTime import DateTime
from Products.PluginIndexes.DateIndex.tests.test_DateIndex \
import _getEastern
before = datetime(2009, 7, 11, 0, 0, tzinfo=_getEastern())
start = datetime(2009, 7, 13, 5, 15, tzinfo=_getEastern())
between = datetime(2009, 7, 13, 5, 45, tzinfo=_getEastern())
stop = datetime(2009, 7, 13, 6, 30, tzinfo=_getEastern())
after = datetime(2009, 7, 14, 0, 0, tzinfo=_getEastern())
dummy = Dummy('test', start, stop)
index = self._makeOne( 'work', 'start', 'stop' )
index.index_object(0, dummy)
self.assertEqual(index.getEntryForObject(0),
(DateTime(start).millis() / 60000,
DateTime(stop).millis() / 60000))
results, used = index._apply_index( { 'work' : before } )
self.assertEqual(len(results), 0)
results, used = index._apply_index( { 'work' : start } )
self.assertEqual(len(results), 1)
results, used = index._apply_index( { 'work' : between } )
self.assertEqual(len(results), 1)
results, used = index._apply_index( { 'work' : stop } )
self.assertEqual(len(results), 1)
results, used = index._apply_index( { 'work' : after } )
self.assertEqual(len(results), 0)
def test_datetime_naive_timezone(self):
from datetime import datetime
from DateTime.DateTime import DateTime
from Products.PluginIndexes.DateIndex.DateIndex import Local
before = datetime(2009, 7, 11, 0, 0)
start = datetime(2009, 7, 13, 5, 15)
start_local = datetime(2009, 7, 13, 5, 15, tzinfo=Local)
between = datetime(2009, 7, 13, 5, 45)
stop = datetime(2009, 7, 13, 6, 30)
stop_local = datetime(2009, 7, 13, 6, 30, tzinfo=Local)
after = datetime(2009, 7, 14, 0, 0)
dummy = Dummy('test', start, stop)
index = self._makeOne( 'work', 'start', 'stop' )
index.index_object(0, dummy)
self.assertEqual(index.getEntryForObject(0),
(DateTime(start_local).millis() / 60000,
DateTime(stop_local).millis() / 60000))
results, used = index._apply_index( { 'work' : before } )
self.assertEqual(len(results), 0)
results, used = index._apply_index( { 'work' : start } )
self.assertEqual(len(results), 1)
results, used = index._apply_index( { 'work' : between } )
self.assertEqual(len(results), 1)
results, used = index._apply_index( { 'work' : stop } )
self.assertEqual(len(results), 1)
results, used = index._apply_index( { 'work' : after } )
self.assertEqual(len(results), 0)
def test_suite():
suite = unittest.TestSuite()
suite.addTest( unittest.makeSuite( DRI_Tests ) )
return suite
src/Products/PluginIndexes/FieldIndex/FieldIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Simple column indices.
"""
from App.special_dtml import DTMLFile
from Products.PluginIndexes.common.UnIndex import UnIndex
class FieldIndex(UnIndex):
"""Index for simple fields.
"""
meta_type="FieldIndex"
manage_options= (
{'label': 'Settings', 'action': 'manage_main'},
{'label': 'Browse', 'action': 'manage_browse'},
)
query_options = ["query","range"]
manage = manage_main = DTMLFile('dtml/manageFieldIndex', globals())
manage_main._setName('manage_main')
manage_browse = DTMLFile('../dtml/browseIndex', globals())
manage_addFieldIndexForm = DTMLFile('dtml/addFieldIndex', globals())
def manage_addFieldIndex(self, id, extra=None,
REQUEST=None, RESPONSE=None, URL3=None):
"""Add a field index"""
return self.manage_addIndex(id, 'FieldIndex', extra=extra, \
REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)
src/Products/PluginIndexes/FieldIndex/dtml/addFieldIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var "manage_form_title(this(), _,
form_title='Add FieldIndex',
)">
<p class="form-help">
<strong>Field Indexes</strong> treat the value of an objects attributes
atomically, and can be used, for example, to track only a certain subset
of object values, such as 'meta_type'.
</p>
<form action="manage_addFieldIndex" method="post" enctype="multipart/form-data">
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Id
</div>
</td>
<td align="left" valign="top">
<input type="text" name="id" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-label">
Indexed attributes
</div>
</td>
<td align="left" valign="top">
<input type="text" name="extra.indexed_attrs:record:string" size="40" />
<em>attribute1,attribute2,...</em> or leave empty
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-optional">
Type
</div>
</td>
<td align="left" valign="top">
Field Index
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/FieldIndex/dtml/manageFieldIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p class="form-help">
Objects indexed: <dtml-var numObjects>
<br>
Distinct values: <dtml-var indexSize>
</p>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/FieldIndex/tests/__init__.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
# This file is needed to make this a package.
src/Products/PluginIndexes/FieldIndex/tests/testFieldIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""FieldIndex unit tests.
"""
import unittest
import Zope2
Zope2.startup()
from Products.PluginIndexes.FieldIndex.FieldIndex import FieldIndex
class Dummy:
def __init__( self, foo ):
self._foo = foo
def foo( self ):
return self._foo
def __str__( self ):
return '<Dummy: %s>' % self._foo
__repr__ = __str__
class FieldIndexTests(unittest.TestCase):
"""Test FieldIndex objects.
"""
def setUp( self ):
"""
"""
self._index = FieldIndex( 'foo' )
self._marker = []
self._values = [ ( 0, Dummy( 'a' ) )
, ( 1, Dummy( 'ab' ) )
, ( 2, Dummy( 'abc' ) )
, ( 3, Dummy( 'abca' ) )
, ( 4, Dummy( 'abcd' ) )
, ( 5, Dummy( 'abce' ) )
, ( 6, Dummy( 'abce' ) )
, ( 7, Dummy( 0 ) ) # Collector #1959
, ( 8, Dummy(None) )]
self._forward = {}
self._backward = {}
for k, v in self._values:
self._backward[k] = v
keys = self._forward.get( v, [] )
self._forward[v] = keys
self._noop_req = { 'bar': 123 }
self._request = { 'foo': 'abce' }
self._min_req = { 'foo': {'query': 'abc'
, 'range': 'min'}
}
self._max_req = { 'foo': {'query': 'abc'
, 'range': 'max' }
}
self._range_req = { 'foo': {'query': ( 'abc', 'abcd' )
, 'range': 'min:max' }
}
self._zero_req = { 'foo': 0 }
self._none_req = { 'foo': None }
def tearDown( self ):
"""
"""
def _populateIndex( self ):
for k, v in self._values:
self._index.index_object( k, v )
def _checkApply( self, req, expectedValues ):
result, used = self._index._apply_index( req )
if hasattr(result, 'keys'):
result = result.keys()
assert used == ( 'foo', )
assert len( result ) == len( expectedValues ), \
'%s | %s' % ( map( None, result ), expectedValues )
for k, v in expectedValues:
assert k in result
def test_interfaces(self):
from Products.PluginIndexes.interfaces import IPluggableIndex
from Products.PluginIndexes.interfaces import ISortIndex
from Products.PluginIndexes.interfaces import IUniqueValueIndex
from zope.interface.verify import verifyClass
verifyClass(IPluggableIndex, FieldIndex)
verifyClass(ISortIndex, FieldIndex)
verifyClass(IUniqueValueIndex, FieldIndex)
def testEmpty( self ):
"Test an empty FieldIndex."
assert len( self._index ) == 0
assert len( self._index.referencedObjects() ) == 0
self.assertEqual(self._index.numObjects(), 0)
assert self._index.getEntryForObject( 1234 ) is None
assert ( self._index.getEntryForObject( 1234, self._marker )
is self._marker )
self._index.unindex_object( 1234 ) # nothrow
assert self._index.hasUniqueValuesFor( 'foo' )
assert not self._index.hasUniqueValuesFor( 'bar' )
assert len( self._index.uniqueValues( 'foo' ) ) == 0
assert self._index._apply_index( self._noop_req ) is None
self._checkApply( self._request, [] )
self._checkApply( self._min_req, [] )
self._checkApply( self._max_req, [] )
self._checkApply( self._range_req, [] )
def testPopulated( self ):
""" Test a populated FieldIndex """
self._populateIndex()
values = self._values
assert len( self._index ) == len( values )-1 #'abce' is duplicate
assert len( self._index.referencedObjects() ) == len( values )
self.assertEqual(self._index.indexSize(), len( values )-1)
assert self._index.getEntryForObject( 1234 ) is None
assert ( self._index.getEntryForObject( 1234, self._marker )
is self._marker )
self._index.unindex_object( 1234 ) # nothrow
for k, v in values:
assert self._index.getEntryForObject( k ) == v.foo()
assert len( self._index.uniqueValues( 'foo' ) ) == len( values )-1
assert self._index._apply_index( self._noop_req ) is None
self._checkApply( self._request, values[ -4:-2 ] )
self._checkApply( self._min_req, values[ 2:-2 ] )
self._checkApply( self._max_req, values[ :3 ] + values[ -2: ] )
self._checkApply( self._range_req, values[ 2:5 ] )
def testZero( self ):
""" Make sure 0 gets indexed """
self._populateIndex()
values = self._values
self._checkApply( self._zero_req, values[ -2:-1 ] )
assert 0 in self._index.uniqueValues( 'foo' )
def testNone(self):
""" make sure None gets indexed """
self._populateIndex()
values = self._values
self._checkApply(self._none_req, values[-1:])
assert None in self._index.uniqueValues('foo')
def testReindex( self ):
self._populateIndex()
result, used = self._index._apply_index( {'foo':'abc'} )
assert list(result)==[2]
assert self._index.keyForDocument(2)=='abc'
d = Dummy('world')
self._index.index_object(2,d)
result, used = self._index._apply_index( {'foo':'world'} )
assert list(result)==[2]
assert self._index.keyForDocument(2)=='world'
del d._foo
self._index.index_object(2,d)
result, used = self._index._apply_index( {'foo':'world'} )
assert list(result)==[]
try:
should_not_be = self._index.keyForDocument(2)
except KeyError:
# As expected, we deleted that attribute.
pass
else:
# before Collector #291 this would be 'world'
raise ValueError(repr(should_not_be))
def testRange(self):
"""Test a range search"""
index = FieldIndex( 'foo' )
for i in range(100):
index.index_object(i, Dummy(i%10))
record = { 'foo' : { 'query' : [-99, 3]
, 'range' : 'min:max'
}
}
r=index._apply_index( record )
assert tuple(r[1])==('foo',), r[1]
r=list(r[0].keys())
expect=[
0, 1, 2, 3, 10, 11, 12, 13, 20, 21, 22, 23, 30, 31, 32, 33,
40, 41, 42, 43, 50, 51, 52, 53, 60, 61, 62, 63, 70, 71, 72, 73,
80, 81, 82, 83, 90, 91, 92, 93
]
assert r==expect, r
#
# Make sure that range tests with incompatible paramters
# don't return empty sets.
#
record[ 'foo' ][ 'operator' ] = 'and'
r2, ignore = index._apply_index( record )
r2 = list( r2.keys() )
assert r2 == r
def test_suite():
return unittest.TestSuite((
unittest.makeSuite(FieldIndexTests),
))
src/Products/PluginIndexes/KeywordIndex/KeywordIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Keyword index.
"""
from logging import getLogger
from BTrees.OOBTree import difference
from BTrees.OOBTree import OOSet
from App.special_dtml import DTMLFile
from Products.PluginIndexes.common import safe_callable
from Products.PluginIndexes.common.UnIndex import UnIndex
LOG = getLogger('Zope.KeywordIndex')
class KeywordIndex(UnIndex):
"""Like an UnIndex only it indexes sequences of items.
Searches match any keyword.
This should have an _apply_index that returns a relevance score
"""
meta_type="KeywordIndex"
manage_options= (
{'label': 'Settings', 'action': 'manage_main'},
{'label': 'Browse', 'action': 'manage_browse'},
)
query_options = ("query","operator", "range")
def _index_object(self, documentId, obj, threshold=None, attr=''):
""" index an object 'obj' with integer id 'i'
Ideally, we've been passed a sequence of some sort that we
can iterate over. If however, we haven't, we should do something
useful with the results. In the case of a string, this means
indexing the entire string as a keyword."""
# First we need to see if there's anything interesting to look at
# self.id is the name of the index, which is also the name of the
# attribute we're interested in. If the attribute is callable,
# we'll do so.
newKeywords = self._get_object_keywords(obj, attr)
oldKeywords = self._unindex.get(documentId, None)
if oldKeywords is None:
# we've got a new document, let's not futz around.
try:
for kw in newKeywords:
self.insertForwardIndexEntry(kw, documentId)
if newKeywords:
self._unindex[documentId] = list(newKeywords)
except TypeError:
return 0
else:
# we have an existing entry for this document, and we need
# to figure out if any of the keywords have actually changed
if type(oldKeywords) is not OOSet:
oldKeywords = OOSet(oldKeywords)
newKeywords = OOSet(newKeywords)
fdiff = difference(oldKeywords, newKeywords)
rdiff = difference(newKeywords, oldKeywords)
if fdiff or rdiff:
# if we've got forward or reverse changes
if newKeywords:
self._unindex[documentId] = list(newKeywords)
else:
del self._unindex[documentId]
if fdiff:
self.unindex_objectKeywords(documentId, fdiff)
if rdiff:
for kw in rdiff:
self.insertForwardIndexEntry(kw, documentId)
return 1
def _get_object_keywords(self, obj, attr):
newKeywords = getattr(obj, attr, ())
if safe_callable(newKeywords):
try:
newKeywords = newKeywords()
except AttributeError:
return ()
if not newKeywords:
return ()
elif isinstance(newKeywords, basestring): #Python 2.1 compat isinstance
return (newKeywords,)
else:
unique = {}
try:
for k in newKeywords:
unique[k] = None
except TypeError:
# Not a sequence
return (newKeywords,)
else:
return unique.keys()
def unindex_objectKeywords(self, documentId, keywords):
""" carefully unindex the object with integer id 'documentId'"""
if keywords is not None:
for kw in keywords:
self.removeForwardIndexEntry(kw, documentId)
def unindex_object(self, documentId):
""" carefully unindex the object with integer id 'documentId'"""
keywords = self._unindex.get(documentId, None)
self.unindex_objectKeywords(documentId, keywords)
try:
del self._unindex[documentId]
except KeyError:
LOG.debug('Attempt to unindex nonexistent'
' document id %s' % documentId)
manage = manage_main = DTMLFile('dtml/manageKeywordIndex', globals())
manage_main._setName('manage_main')
manage_browse = DTMLFile('../dtml/browseIndex', globals())
manage_addKeywordIndexForm = DTMLFile('dtml/addKeywordIndex', globals())
def manage_addKeywordIndex(self, id, extra=None,
REQUEST=None, RESPONSE=None, URL3=None):
"""Add a keyword index"""
return self.manage_addIndex(id, 'KeywordIndex', extra=extra, \
REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)
src/Products/PluginIndexes/KeywordIndex/dtml/addKeywordIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var "manage_form_title(this(), _,
form_title='Add KeywordIndex',
)">
<p class="form-help">
<strong>Keyword Indexes</strong> index a sequence of objects that act as
'keywords' for an object. A Keyword Index will return any objects
that have one or more keywords specified in a search query.
</p>
<form action="manage_addKeywordIndex" method="post" enctype="multipart/form-data">
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Id
</div>
</td>
<td align="left" valign="top">
<input type="text" name="id" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-label">
Indexed attributes
</div>
</td>
<td align="left" valign="top">
<input type="text" name="extra.indexed_attrs:record:string" size="40" />
<em>attribute1,attribute2,...</em> or leave empty
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-optional">
Type
</div>
</td>
<td align="left" valign="top">
Keyword Index
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/KeywordIndex/dtml/manageKeywordIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p class="form-help">
Objects indexed: <dtml-var numObjects>
<br>
Distinct values: <dtml-var indexSize>
</p>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/KeywordIndex/tests/__init__.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
# This file is needed to make this a package.
src/Products/PluginIndexes/KeywordIndex/tests/testKeywordIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""KeywordIndex unit tests.
"""
import unittest
import Zope2
Zope2.startup()
from Products.PluginIndexes.KeywordIndex.KeywordIndex import KeywordIndex
class Dummy:
def __init__( self, foo ):
self._foo = foo
def foo( self ):
return self._foo
def __str__( self ):
return '<Dummy: %s>' % self._foo
__repr__ = __str__
def sortedUnique(seq):
unique = {}
for i in seq:
unique[i] = None
unique = unique.keys()
unique.sort()
return unique
class TestKeywordIndex( unittest.TestCase ):
"""
Test KeywordIndex objects.
"""
_old_log_write = None
def setUp( self ):
"""
"""
self._index = KeywordIndex( 'foo' )
self._marker = []
self._values = [ ( 0, Dummy( ['a'] ) )
, ( 1, Dummy( ['a','b'] ) )
, ( 2, Dummy( ['a','b','c'] ) )
, ( 3, Dummy( ['a','b','c','a'] ) )
, ( 4, Dummy( ['a', 'b', 'c', 'd'] ) )
, ( 5, Dummy( ['a', 'b', 'c', 'e'] ) )
, ( 6, Dummy( ['a', 'b', 'c', 'e', 'f'] ))
, ( 7, Dummy( [0] ) )
]
self._noop_req = { 'bar': 123 }
self._all_req = { 'foo': ['a'] }
self._some_req = { 'foo': ['e'] }
self._overlap_req = { 'foo': ['c', 'e'] }
self._string_req = {'foo': 'a'}
self._zero_req = { 'foo': [0] }
def tearDown( self ):
"""
"""
def _populateIndex( self ):
for k, v in self._values:
self._index.index_object( k, v )
def _checkApply( self, req, expectedValues ):
result, used = self._index._apply_index( req )
assert used == ( 'foo', )
assert len(result) == len( expectedValues ), \
'%s | %s' % ( map( None, result ),
map(lambda x: x[0], expectedValues ))
if hasattr(result, 'keys'): result=result.keys()
for k, v in expectedValues:
assert k in result
def test_interfaces(self):
from Products.PluginIndexes.interfaces import IPluggableIndex
from Products.PluginIndexes.interfaces import ISortIndex
from Products.PluginIndexes.interfaces import IUniqueValueIndex
from zope.interface.verify import verifyClass
verifyClass(IPluggableIndex, KeywordIndex)
verifyClass(ISortIndex, KeywordIndex)
verifyClass(IUniqueValueIndex, KeywordIndex)
def testAddObjectWOKeywords(self):
try:
self._populateIndex()
self._index.index_object(999, None)
finally:
pass
def testEmpty( self ):
assert len( self._index ) == 0
assert len( self._index.referencedObjects() ) == 0
self.assertEqual(self._index.numObjects(), 0)
assert self._index.getEntryForObject( 1234 ) is None
assert ( self._index.getEntryForObject( 1234, self._marker )
is self._marker ), self._index.getEntryForObject(1234)
self._index.unindex_object( 1234 ) # nothrow
assert self._index.hasUniqueValuesFor( 'foo' )
assert not self._index.hasUniqueValuesFor( 'bar' )
assert len( self._index.uniqueValues( 'foo' ) ) == 0
assert self._index._apply_index( self._noop_req ) is None
self._checkApply( self._all_req, [] )
self._checkApply( self._some_req, [] )
self._checkApply( self._overlap_req, [] )
self._checkApply( self._string_req, [] )
def testPopulated( self ):
self._populateIndex()
values = self._values
#assert len( self._index ) == len( values )
assert len( self._index.referencedObjects() ) == len( values )
assert self._index.getEntryForObject( 1234 ) is None
assert ( self._index.getEntryForObject( 1234, self._marker )
is self._marker )
self._index.unindex_object( 1234 ) # nothrow
self.assertEqual(self._index.indexSize(), len( values )-1)
for k, v in values:
entry = self._index.getEntryForObject( k )
entry.sort()
kw = sortedUnique(v.foo())
self.assertEqual(entry, kw)
assert len( self._index.uniqueValues( 'foo' ) ) == len( values )-1
assert self._index._apply_index( self._noop_req ) is None
self._checkApply( self._all_req, values[:-1])
self._checkApply( self._some_req, values[ 5:7 ] )
self._checkApply( self._overlap_req, values[2:7] )
self._checkApply( self._string_req, values[:-1] )
def testZero( self ):
self._populateIndex()
values = self._values
self._checkApply( self._zero_req, values[ -1: ] )
assert 0 in self._index.uniqueValues( 'foo' )
def testReindexChange(self):
self._populateIndex()
expected = Dummy(['x', 'y'])
self._index.index_object(6, expected)
result, used = self._index._apply_index({'foo': ['x', 'y']})
result=result.keys()
assert len(result) == 1
assert result[0] == 6
result, used = self._index._apply_index(
{'foo': ['a', 'b', 'c', 'e', 'f']}
)
result = result.keys()
assert 6 not in result
def testReindexNoChange(self):
self._populateIndex()
expected = Dummy(['foo', 'bar'])
self._index.index_object(8, expected)
result, used = self._index._apply_index(
{'foo': ['foo', 'bar']})
result = result.keys()
assert len(result) == 1
assert result[0] == 8
self._index.index_object(8, expected)
result, used = self._index._apply_index(
{'foo': ['foo', 'bar']})
result = result.keys()
assert len(result) == 1
assert result[0] == 8
def testIntersectionWithRange(self):
# Test an 'and' search, ensuring that 'range' doesn't mess it up.
self._populateIndex()
record = { 'foo' : { 'query' : [ 'e', 'f' ]
, 'operator' : 'and'
}
}
self._checkApply( record, self._values[6:7] )
#
# Make sure that 'and' tests with incompatible paramters
# don't return empty sets.
#
record[ 'foo' ][ 'range' ] = 'min:max'
self._checkApply( record, self._values[6:7] )
def testDuplicateKeywords(self):
try:
self._index.index_object(0, Dummy(['a', 'a', 'b', 'b']))
self._index.unindex_object(0)
finally:
pass
def testCollectorIssue889(self) :
# Test that collector issue 889 is solved
values = self._values
nonexistent = 'foo-bar-baz'
self._populateIndex()
# make sure key is not indexed
result = self._index._index.get(nonexistent, self._marker)
assert result is self._marker
# patched _apply_index now works as expected
record = {'foo' : { 'query' : [nonexistent]
, 'operator' : 'and'}
}
self._checkApply(record, [])
record = {'foo' : { 'query' : [nonexistent, 'a']
, 'operator' : 'and'}
}
# and does not break anything
self._checkApply(record, [])
record = {'foo' : { 'query' : ['d']
, 'operator' : 'and'}
}
self._checkApply(record, values[4:5])
record = {'foo' : { 'query' : ['a', 'e']
, 'operator' : 'and'}
}
self._checkApply(record, values[5:7])
def test_noindexing_when_noattribute(self):
to_index = Dummy(['hello'])
self._index._index_object(10, to_index, attr='UNKNOWN')
self.assertFalse(self._index._unindex.get(10))
self.assertFalse(self._index.getEntryForObject(10))
def test_noindexing_when_raising_attribute(self):
class FauxObject:
def foo(self):
raise AttributeError
to_index = FauxObject()
self._index._index_object(10, to_index, attr='foo')
self.assertFalse(self._index._unindex.get(10))
self.assertFalse(self._index.getEntryForObject(10))
def test_value_removes(self):
to_index = Dummy(['hello'])
self._index._index_object(10, to_index, attr='foo')
self.assertTrue(self._index._unindex.get(10))
to_index = Dummy('')
self._index._index_object(10, to_index, attr='foo')
self.assertFalse(self._index._unindex.get(10))
def test_suite():
suite = unittest.TestSuite()
suite.addTest( unittest.makeSuite( TestKeywordIndex ) )
return suite
src/Products/PluginIndexes/PathIndex/PathIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Path index.
"""
from logging import getLogger
from App.special_dtml import DTMLFile
from OFS.SimpleItem import SimpleItem
from BTrees.IIBTree import IITreeSet
from BTrees.IIBTree import IISet
from BTrees.IIBTree import intersection
from BTrees.IIBTree import multiunion
from BTrees.IIBTree import union
from BTrees.IOBTree import IOBTree
from BTrees.OOBTree import OOBTree
from BTrees.Length import Length
from Persistence import Persistent
from zope.interface import implements
from Products.PluginIndexes.common import safe_callable
from Products.PluginIndexes.common.util import parseIndexRequest
from Products.PluginIndexes.interfaces import IPathIndex
from Products.PluginIndexes.interfaces import ISortIndex
from Products.PluginIndexes.interfaces import IUniqueValueIndex
LOG = getLogger('Zope.PathIndex')
class PathIndex(Persistent, SimpleItem):
"""Index for paths returned by getPhysicalPath.
A path index stores all path components of the physical path of an object.
Internal datastructure:
- a physical path of an object is split into its components
- every component is kept as a key of a OOBTree in self._indexes
- the value is a mapping 'level of the path component' to
'all docids with this path component on this level'
"""
implements(IPathIndex, IUniqueValueIndex, ISortIndex)
meta_type="PathIndex"
query_options = ('query', 'level', 'operator')
manage_options= (
{'label': 'Settings', 'action': 'manage_main'},
)
def __init__(self,id,caller=None):
self.id = id
self.operators = ('or','and')
self.useOperator = 'or'
self.clear()
def __len__(self):
return self._length()
# IPluggableIndex implementation
def getEntryForObject(self, docid, default=None):
""" See IPluggableIndex.
"""
try:
return self._unindex[docid]
except KeyError:
return default
def getIndexSourceNames(self):
""" See IPluggableIndex.
"""
return (self.id, 'getPhysicalPath', )
def index_object(self, docid, obj ,threshold=100):
""" See IPluggableIndex.
"""
f = getattr(obj, self.id, None)
if f is not None:
if safe_callable(f):
try:
path = f()
except AttributeError:
return 0
else:
path = f
if not isinstance(path, (str, tuple)):
raise TypeError('path value must be string or tuple of strings')
else:
try:
path = obj.getPhysicalPath()
except AttributeError:
return 0
if isinstance(path, (list, tuple)):
path = '/'+ '/'.join(path[1:])
comps = filter(None, path.split('/'))
if not self._unindex.has_key(docid):
self._length.change(1)
for i in range(len(comps)):
self.insertEntry(comps[i], docid, i)
self._unindex[docid] = path
return 1
def unindex_object(self, docid):
""" See IPluggableIndex.
"""
if docid not in self._unindex:
LOG.debug('Attempt to unindex nonexistent document with id %s'
% docid)
return
comps = self._unindex[docid].split('/')
for level in range(len(comps[1:])):
comp = comps[level+1]
try:
self._index[comp][level].remove(docid)
if not self._index[comp][level]:
del self._index[comp][level]
if not self._index[comp]:
del self._index[comp]
except KeyError:
LOG.debug('Attempt to unindex document with id %s failed'
% docid)
self._length.change(-1)
del self._unindex[docid]
def _apply_index(self, request):
""" See IPluggableIndex.
o Unpacks args from catalog and mapps onto '_search'.
"""
record = parseIndexRequest(request, self.id, self.query_options)
if record.keys is None:
return None
level = record.get("level", 0)
operator = record.get('operator', self.useOperator).lower()
# depending on the operator we use intersection of union
if operator == "or":
set_func = union
else:
set_func = intersection
res = None
for k in record.keys:
rows = self._search(k,level)
res = set_func(res,rows)
if res:
return res, (self.id,)
else:
return IISet(), (self.id,)
def numObjects(self):
""" See IPluggableIndex.
"""
return len(self._unindex)
def indexSize(self):
""" See IPluggableIndex.
"""
return len(self)
def clear(self):
""" See IPluggableIndex.
"""
self._depth = 0
self._index = OOBTree()
self._unindex = IOBTree()
self._length = Length(0)
# IUniqueValueIndex implementation
def hasUniqueValuesFor(self, name):
""" See IUniqueValueIndex.
"""
return name == self.id
def uniqueValues(self, name=None, withLength=0):
""" See IUniqueValueIndex.
"""
if name in (None, self.id, 'getPhysicalPath'):
if withLength:
for key in self._index:
yield key, len(self._search(key, -1))
else:
for key in self._index.keys():
yield key
# ISortIndex implementation
def keyForDocument(self, documentId):
""" See ISortIndex.
"""
return self._unindex.get(documentId)
def documentToKeyMap(self):
""" See ISortIndex.
"""
return self._unindex
# IPathIndex implementation.
def insertEntry(self, comp, id, level):
""" See IPathIndex
"""
if not self._index.has_key(comp):
self._index[comp] = IOBTree()
if not self._index[comp].has_key(level):
self._index[comp][level] = IITreeSet()
self._index[comp][level].insert(id)
if level > self._depth:
self._depth = level
# Helper methods
def _search(self, path, default_level=0):
""" Perform the actual search.
``path``
a string representing a relative URL, or a part of a relative URL,
or a tuple ``(path, level)``. In the first two cases, use
``default_level`` as the level for the search.
``default_level``
the level to use for non-tuple queries.
``level >= 0`` => match ``path`` only at the given level.
``level < 0`` => match ``path`` at *any* level
"""
if isinstance(path, str):
level = default_level
else:
level = int(path[1])
path = path[0]
if level < 0:
# Search at every level, return the union of all results
return multiunion(
[self._search(path, level)
for level in xrange(self._depth + 1)])
comps = filter(None, path.split('/'))
if level + len(comps) - 1 > self._depth:
# Our search is for a path longer than anything in the index
return IISet()
if len(comps) == 0:
return IISet(self._unindex.keys())
results = None
for i, comp in reversed(list(enumerate(comps))):
if not self._index.get(comp, {}).has_key(level+i): return IISet()
results = intersection(results, self._index[comp][level+i])
return results
manage = manage_main = DTMLFile('dtml/managePathIndex', globals())
manage_main._setName('manage_main')
manage_addPathIndexForm = DTMLFile('dtml/addPathIndex', globals())
def manage_addPathIndex(self, id, REQUEST=None, RESPONSE=None, URL3=None):
"""Add a path index"""
return self.manage_addIndex(id, 'PathIndex', extra=None, \
REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)
src/Products/PluginIndexes/PathIndex/PathIndex.txt deleted 100644 → 0
View file @ 03a5df08
he purpose of a PathIndex is to index Zope objects
based on their physical path. This is very similiar
to a substring search on strings.
How it works
Assume we have to index an object with id=xxx and
the physical path '/zoo/animals/africa/tiger.doc'.
We split the path into its components and keep track
of the level of every component. Inside the index we
store pairs(component,level) and the ids of the
documents::
(component,level) id of document
-----------------------------------------
('zoo',0) xxx
('animals',1) xxx
('africa',2) xxx
Note that we do not store the id of the objects itself
inside the path index.
Searching with the PathIndex
The PathIndex allows you to search for all object ids
whose objects match a physical path query. The query
is split into components and matched against the index.
E.g. '/zoo/animals' will match in the example above
but not '/zoo1/animals'. The default behaviour is to
start matching at level 0. To start matching on another
level on can specify an additional level parameter
(see API)
API
'query' -- A single or list of Path component(s) to
be searched.
'level' -- level to start searching (optional,default: 0).
If level=-1 we search through all levels.
'operator' -- either 'or' or 'and' (optional, default: 'or')
Example
Objects with the following ids and physical path should
be stored in the ZCatalog 'MyCAT'::
id physical path
----------------------------
1 /aa/bb/aa/1.txt
2 /aa/bb/bb/2.txt
3 /aa/bb/cc/3.txt
4 /bb/bb/aa/4.txt
5 /bb/bb/bb/5.txt
6 /bb/bb/cc/6.txt
7 /cc/bb/aa/7.txt
8 /cc/bb/bb/8.txt
9 /cc/bb/cc/9.txt
Query found ids
-------------------------------------------
query='/aa/bb',level=0 [1,2,3]
query='/bb/bb',level=0 [4,5,6]
query='/bb/bb',level=1 [2,5,8]
query='/bb/bb',level=-1 [2,4,5,6,8]
query='/xx' ,level=-1 []
src/Products/PluginIndexes/PathIndex/dtml/addPathIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var "manage_form_title(this(), _,
form_title='Add PathIndex',
)">
<p class="form-help">
A <em>PathIndex</em> indexes the physical path of all objects inside
a catalog. It allows you to search for objects beginning or containing
a special path component or a set of path component. A path component
is defined as <em>/&lt;component1&gt;/&lt;component2&gt;/..../&lt;object_id&gt;
</em>.
</p>
<form action="manage_addPathIndex" method="post" enctype="multipart/form-data">
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Id
</div>
</td>
<td align="left" valign="top">
<input type="text" name="id" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-optional">
Type
</div>
</td>
<td align="left" valign="top">
PathIndex
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/PathIndex/dtml/managePathIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p class="form-help">
Objects indexed: <dtml-var numObjects>
<br>
Distinct values: <dtml-var indexSize>
</p>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/PathIndex/tests/__init__.py deleted 100644 → 0
View file @ 03a5df08
# This file is needed to make this directory a package.
src/Products/PluginIndexes/TopicIndex/FilteredSet.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Filtered set.
"""
from logging import getLogger
import sys
from BTrees.IIBTree import IITreeSet
from Persistence import Persistent
from RestrictedPython.Eval import RestrictionCapableEval
from ZODB.POSException import ConflictError
from zope.interface import implements
from Products.PluginIndexes.interfaces import IFilteredSet
LOG = getLogger('Zope.TopicIndex.FilteredSet')
class FilteredSetBase(Persistent):
# A pre-calculated result list based on an expression.
implements(IFilteredSet)
def __init__(self, id, expr):
self.id = id
self.expr = expr
self.clear()
def clear(self):
self.ids = IITreeSet()
def index_object(self, documentId, obj):
raise RuntimeError,'index_object not defined'
def unindex_object(self,documentId):
try: self.ids.remove(documentId)
except KeyError: pass
def getId(self):
return self.id
def getExpression(self):
# Get the expression.
return self.expr
def getIds(self):
# Get the IDs of all objects for which the expression is True.
return self.ids
def getType(self):
return self.meta_type
def setExpression(self, expr):
# Set the expression.
self.expr = expr
def __repr__(self):
return '%s: (%s) %s' % (self.id,self.expr,map(None,self.ids))
__str__ = __repr__
class PythonFilteredSet(FilteredSetBase):
meta_type = 'PythonFilteredSet'
def index_object(self, documentId, o):
try:
if RestrictionCapableEval(self.expr).eval({'o': o}):
self.ids.insert(documentId)
else:
try:
self.ids.remove(documentId)
except KeyError:
pass
except ConflictError:
raise
except:
LOG.warn('eval() failed Object: %s, expr: %s' %\
(o.getId(),self.expr), exc_info=sys.exc_info())
def factory(f_id, f_type, expr):
""" factory function for FilteredSets """
if f_type=='PythonFilteredSet':
return PythonFilteredSet(f_id, expr)
else:
raise TypeError,'unknown type for FilteredSets: %s' % f_type
src/Products/PluginIndexes/TopicIndex/README.txt deleted 100644 → 0
View file @ 03a5df08
TopicIndex
Reference: http://dev.zope.org/Wikis/DevSite/Proposals/TopicIndexes
A TopicIndex is a container for so-called FilteredSet. A FilteredSet
consists of an expression and a set of internal ZCatalog document
identifiers that represent a pre-calculated result list for performance
reasons. Instead of executing the same query on a ZCatalog multiple times
it is much faster to use a TopicIndex instead.
Building up FilteredSets happens on the fly when objects are cataloged
and uncatalogued. Every indexed object is evaluated against the expressions
of every FilteredSet. An object is added to a FilteredSet if the expression
with the object evaluates to 1. Uncatalogued objects are removed from the
FilteredSet.
Types of FilteredSet
PythonFilteredSet
A PythonFilteredSet evaluates using the eval() function inside the
context of the FilteredSet class. The object to be indexes must
be referenced inside the expression using "o.".
Examples::
"o.meta_type=='DTML Method'"
Queries on TopicIndexes
A TopicIndex is queried in the same way as other ZCatalog Indexes and supports
usage of the 'operator' parameter to specify how to combine search results.
API
The TopicIndex implements the API for pluggable Indexes.
Additionally it provides the following functions to manage FilteredSets
-- addFilteredSet(Id, filterType, expression):
-- Id: unique Id for the FilteredSet
-- filterType: 'PythonFilteredSet'
-- expression: Python expression
-- delFilteredSet(Id):
-- clearFilteredSet(Id):
src/Products/PluginIndexes/TopicIndex/TopicIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Topic index.
"""
from logging import getLogger
from App.special_dtml import DTMLFile
from BTrees.IIBTree import IITreeSet
from BTrees.IIBTree import intersection
from BTrees.IIBTree import union
from BTrees.OOBTree import OOBTree
from OFS.SimpleItem import SimpleItem
from Persistence import Persistent
from zope.interface import implements
from Products.PluginIndexes.common.util import parseIndexRequest
from Products.PluginIndexes.interfaces import IPluggableIndex
from Products.PluginIndexes.interfaces import ITopicIndex
from Products.PluginIndexes.TopicIndex.FilteredSet import factory
_marker = []
LOG = getLogger('Zope.TopicIndex')
class TopicIndex(Persistent, SimpleItem):
"""A TopicIndex maintains a set of FilteredSet objects.
Every FilteredSet object consists of an expression and and IISet with all
Ids of indexed objects that eval with this expression to 1.
"""
implements(ITopicIndex, IPluggableIndex)
meta_type="TopicIndex"
query_options = ('query', 'operator')
manage_options= (
{'label': 'FilteredSets', 'action': 'manage_main'},
)
def __init__(self,id,caller=None):
self.id = id
self.filteredSets = OOBTree()
self.operators = ('or','and')
self.defaultOperator = 'or'
def getId(self):
return self.id
def clear(self):
for fs in self.filteredSets.values():
fs.clear()
def index_object(self, docid, obj ,threshold=100):
""" hook for (Z)Catalog """
for fid, filteredSet in self.filteredSets.items():
filteredSet.index_object(docid,obj)
return 1
def unindex_object(self,docid):
""" hook for (Z)Catalog """
for fs in self.filteredSets.values():
try:
fs.unindex_object(docid)
except KeyError:
LOG.debug('Attempt to unindex document'
' with id %s failed' % docid)
return 1
def numObjects(self):
"""Return the number of indexed objects."""
return "n/a"
def indexSize(self):
"""Return the size of the index in terms of distinct values."""
return "n/a"
def search(self,filter_id):
if self.filteredSets.has_key(filter_id):
return self.filteredSets[filter_id].getIds()
def _apply_index(self, request):
""" hook for (Z)Catalog
'request' -- mapping type (usually {"topic": "..." }
"""
record = parseIndexRequest(request, self.id, self.query_options)
if record.keys is None:
return None
operator = record.get('operator', self.defaultOperator).lower()
if operator == 'or': set_func = union
else: set_func = intersection
res = None
for filter_id in record.keys:
rows = self.search(filter_id)
res = set_func(res,rows)
if res:
return res, (self.id,)
else:
return IITreeSet(), (self.id,)
def uniqueValues(self,name=None, withLength=0):
""" needed to be consistent with the interface """
return self.filteredSets.keys()
def getEntryForObject(self,docid, default=_marker):
""" Takes a document ID and returns all the information we have
on that specific object.
"""
return self.filteredSets.keys()
def addFilteredSet(self, filter_id, typeFilteredSet, expr):
# Add a FilteredSet object.
if self.filteredSets.has_key(filter_id):
raise KeyError,\
'A FilteredSet with this name already exists: %s' % filter_id
self.filteredSets[filter_id] = factory(filter_id,
typeFilteredSet,
expr,
)
def delFilteredSet(self, filter_id):
# Delete the FilteredSet object specified by 'filter_id'.
if not self.filteredSets.has_key(filter_id):
raise KeyError,\
'no such FilteredSet: %s' % filter_id
del self.filteredSets[filter_id]
def clearFilteredSet(self, filter_id):
# Clear the FilteredSet object specified by 'filter_id'.
if not self.filteredSets.has_key(filter_id):
raise KeyError,\
'no such FilteredSet: %s' % filter_id
self.filteredSets[filter_id].clear()
def manage_addFilteredSet(self, filter_id, typeFilteredSet, expr, URL1, \
REQUEST=None,RESPONSE=None):
""" add a new filtered set """
if len(filter_id) == 0: raise RuntimeError,'Length of ID too short'
if len(expr) == 0: raise RuntimeError,'Length of expression too short'
self.addFilteredSet(filter_id, typeFilteredSet, expr)
if RESPONSE:
RESPONSE.redirect(URL1+'/manage_workspace?'
'manage_tabs_message=FilteredSet%20added')
def manage_delFilteredSet(self, filter_ids=[], URL1=None, \
REQUEST=None,RESPONSE=None):
""" delete a list of FilteredSets"""
for filter_id in filter_ids:
self.delFilteredSet(filter_id)
if RESPONSE:
RESPONSE.redirect(URL1+'/manage_workspace?'
'manage_tabs_message=FilteredSet(s)%20deleted')
def manage_saveFilteredSet(self,filter_id, expr, URL1=None,\
REQUEST=None,RESPONSE=None):
""" save expression for a FilteredSet """
self.filteredSets[filter_id].setExpression(expr)
if RESPONSE:
RESPONSE.redirect(URL1+'/manage_workspace?'
'manage_tabs_message=FilteredSet(s)%20updated')
def getIndexSourceNames(self):
""" return names of indexed attributes """
return ('n/a',)
def manage_clearFilteredSet(self, filter_ids=[], URL1=None, \
REQUEST=None,RESPONSE=None):
""" clear a list of FilteredSets"""
for filter_id in filter_ids:
self.clearFilteredSet(filter_id)
if RESPONSE:
RESPONSE.redirect(URL1+'/manage_workspace?'
'manage_tabs_message=FilteredSet(s)%20cleared')
manage = manage_main = DTMLFile('dtml/manageTopicIndex',globals())
manage_main._setName('manage_main')
editFilteredSet = DTMLFile('dtml/editFilteredSet',globals())
manage_addTopicIndexForm = DTMLFile('dtml/addTopicIndex', globals())
def manage_addTopicIndex(self, id, REQUEST=None, RESPONSE=None, URL3=None):
"""Add a TopicIndex"""
return self.manage_addIndex(id, 'TopicIndex', extra=None, \
REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL3)
src/Products/PluginIndexes/TopicIndex/__init__.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
src/Products/PluginIndexes/TopicIndex/dtml/addTopicIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var "manage_form_title(this(), _,
form_title='Add TopicIndex',
)">
<p class="form-help">
A <em>TopicIndex</em> is a container for so-called <em>FilteredSets</em>
that consist of an expression and a set of internal ZCatalog document
identifiers that fulfill this expression. <em>TopicIndexes</em> are
useful for performance reasons when search queries take too long
and pre-calculated result sets offer a better performance.
</p>
<form action="manage_addTopicIndex" method="post" enctype="multipart/form-data">
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Id
</div>
</td>
<td align="left" valign="top">
<input type="text" name="id" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-optional">
Type
</div>
</td>
<td align="left" valign="top">
TopicIndex
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/TopicIndex/dtml/editFilteredSet.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<p>
<dtml-with "filteredSets[filteredSet]">
<form action="manage_saveFilteredSet" method="post" enctype="multipart/form-data">
<input type="hidden" name="filter_id" value="&dtml-getId;" >
<table cellspacing="0" cellpadding="2" border="1" width="90%" align="center">
<tr>
<th colspan="2">Edit FilteredSet</th>
</tr>
<tr>
<th>FilteredSet Id</th>
<td>
&dtml-getId;
</td>
</tr>
<tr>
<th>FilteredSet Type</th>
<td>&dtml-getType;</td>
</tr>
<tr>
<th>FilteredSet Expression</th>
<td>
<textarea name="expr" cols="60" rows="5">&dtml-getExpression;</textarea>
</td>
</tr>
<tr>
<td colspan="2" align="center">
<input class="form-element" type="submit" value=" Save " />
</td>
</tr>
</table>
</form>
</dtml-with>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/TopicIndex/dtml/manageTopicIndex.dtml deleted 100644 → 0
View file @ 03a5df08
<dtml-var manage_page_header>
<dtml-var manage_tabs>
<form action="&dtml-URL1;/" method="post" enctype="multipart/form-data">
<table cellspacing="0" cellpadding="2" border="1" width="90%" align="center">
<tr>
<th colspan="5">
Defined FilteredSets
</th>
</tr>
<dtml-if "_.len(filteredSets.values())>0">
<tr>
<th>&nbsp;</th>
<th>FilteredSet Id</th>
<th>FilteredSet Type</th>
<th>Expression</th>
<th># entries</th>
</tr>
<dtml-in expr="filteredSets.values()">
<dtml-call "REQUEST.set('fs',_['sequence-item'])">
<tr>
<td align="center">
<input type="checkbox" name="filter_ids:list" value="<dtml-var "fs.getId()" html_quote>">
</td>
<td align="center" valign="top">
<div class="form-label">
<a href="editFilteredSet?filteredSet=&dtml-id;">&dtml-getId; </a>
</div>
</td>
<td align="center" valign="top">
<div class="form-label">
&dtml-getType;
</div>
</td>
<td align="left" valign="top">
<div class="form-label">
&dtml-getExpression;
</div>
</td>
<td align="center" valign="top">
<div class="form-label">
<dtml-var "_.len(fs.getIds())">
</div>
</td>
</tr>
</dtml-in>
<tr>
<td colspan="5" align="center">
<input class="form-element" type="submit" name="manage_delFilteredSet:method"
value=" Remove " />
<input class="form-element" type="submit" name="manage_clearFilteredSet:method"
value=" Clear " />
</td>
</tr>
<dtml-else>
<tr>
<td colspan="5" align="center">
<em>no FilteredSets defined </em>
</td>
</tr>
</dtml-if>
</table>
</form>
<hr>
<form action="manage_addFilteredSet" method="post" enctype="multipart/form-data">
<table cellspacing="0" cellpadding="2" border="0">
<tr>
<td align="left" valign="top">
<div class="form-label">
Id for FilteredSet
</div>
</td>
<td align="left" valign="top">
<input type="text" name="filter_id" size="40" />
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-label">
Type of FilteredSet
</div>
</td>
<td align="left" valign="top">
<select name="typeFilteredSet">
<option value="PythonFilteredSet">PythonFilteredSet
<dtml-comment>
<option value="AttributeFilteredSet">AttributeFilteredSet
</dtml-comment>
</select>
</td>
</tr>
<tr>
<td align="left" valign="top">
<div class="form-label">
Expression
</div>
</td>
<td align="left" valign="top">
<textarea type="text" name="expr" cols="60" rows="5"></textarea>
</td>
</tr>
<tr>
<td align="left" valign="top">
</td>
<td align="left" valign="top">
<div class="form-element">
<input class="form-element" type="submit" name="submit"
value=" Add " />
</div>
</td>
</tr>
</table>
</form>
<dtml-var manage_page_footer>
src/Products/PluginIndexes/TopicIndex/tests/__init__.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2003 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
# This file is needed to make this a package.
src/Products/PluginIndexes/TopicIndex/tests/testTopicIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""TopicIndex unit tests.
"""
import unittest
import Zope2
Zope2.startup()
from Products.PluginIndexes.TopicIndex.TopicIndex import TopicIndex
class Obj:
def __init__(self,id,meta_type=''):
self.id = id
self.meta_type = meta_type
def getId(self): return self.id
def getPhysicalPath(self): return self.id
class TestBase(unittest.TestCase):
def _searchAnd(self,query,expected):
return self._search(query,'and',expected)
def _searchOr(self,query,expected):
return self._search(query,'or',expected)
def _search(self,query,operator,expected):
res = self.TI._apply_index({'topic':{'query':query,'operator':operator}})
rows = list(res[0].keys())
rows.sort()
expected.sort()
self.assertEqual(rows,expected,query)
return rows
class TestTopicIndex(TestBase):
def setUp(self):
self.TI = TopicIndex("topic")
self.TI.addFilteredSet("doc1","PythonFilteredSet","o.meta_type=='doc1'")
self.TI.addFilteredSet("doc2","PythonFilteredSet","o.meta_type=='doc2'")
self.TI.index_object(0 , Obj('0',))
self.TI.index_object(1 , Obj('1','doc1'))
self.TI.index_object(2 , Obj('2','doc1'))
self.TI.index_object(3 , Obj('3','doc2'))
self.TI.index_object(4 , Obj('4','doc2'))
self.TI.index_object(5 , Obj('5','doc3'))
self.TI.index_object(6 , Obj('6','doc3'))
def test_interfaces(self):
from Products.PluginIndexes.interfaces import ITopicIndex
from Products.PluginIndexes.interfaces import IPluggableIndex
from zope.interface.verify import verifyClass
verifyClass(ITopicIndex, TopicIndex)
verifyClass(IPluggableIndex, TopicIndex)
def testOr(self):
self._searchOr('doc1',[1,2])
self._searchOr(['doc1'],[1,2])
self._searchOr('doc2',[3,4]),
self._searchOr(['doc2'],[3,4])
self._searchOr(['doc1','doc2'], [1,2,3,4])
def testAnd(self):
self._searchAnd('doc1',[1,2])
self._searchAnd(['doc1'],[1,2])
self._searchAnd('doc2',[3,4])
self._searchAnd(['doc2'],[3,4])
self._searchAnd(['doc1','doc2'],[])
def testRemoval(self):
self.TI.index_object(1, Obj('1','doc2'))
self._searchOr('doc1',[2])
self._searchOr('doc2', [1,3,4])
def test_suite():
return unittest.TestSuite((
unittest.makeSuite(TestTopicIndex),
))
src/Products/PluginIndexes/__init__.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
def initialize(context):
from Products.PluginIndexes.FieldIndex.FieldIndex import FieldIndex
from Products.PluginIndexes.FieldIndex.FieldIndex \
import manage_addFieldIndex
from Products.PluginIndexes.FieldIndex.FieldIndex \
import manage_addFieldIndexForm
context.registerClass(FieldIndex,
permission='Add Pluggable Index',
constructors=(manage_addFieldIndexForm,
manage_addFieldIndex),
icon='www/index.gif',
visibility=None,
)
from Products.PluginIndexes.KeywordIndex.KeywordIndex import KeywordIndex
from Products.PluginIndexes.KeywordIndex.KeywordIndex \
import manage_addKeywordIndex
from Products.PluginIndexes.KeywordIndex.KeywordIndex \
import manage_addKeywordIndexForm
context.registerClass(KeywordIndex,
permission='Add Pluggable Index',
constructors=(manage_addKeywordIndexForm,
manage_addKeywordIndex),
icon='www/index.gif',
visibility=None,
)
from Products.PluginIndexes.TopicIndex.TopicIndex import TopicIndex
from Products.PluginIndexes.TopicIndex.TopicIndex \
import manage_addTopicIndex
from Products.PluginIndexes.TopicIndex.TopicIndex \
import manage_addTopicIndexForm
context.registerClass(TopicIndex,
permission='Add Pluggable Index',
constructors=(manage_addTopicIndexForm,
manage_addTopicIndex),
icon='www/index.gif',
visibility=None,
)
from Products.PluginIndexes.DateIndex.DateIndex import DateIndex
from Products.PluginIndexes.DateIndex.DateIndex \
import manage_addDateIndex
from Products.PluginIndexes.DateIndex.DateIndex \
import manage_addDateIndexForm
context.registerClass(DateIndex,
permission='Add Pluggable Index',
constructors=(manage_addDateIndexForm,
manage_addDateIndex),
icon='www/index.gif',
visibility=None,
)
from Products.PluginIndexes.DateRangeIndex.DateRangeIndex \
import DateRangeIndex
from Products.PluginIndexes.DateRangeIndex.DateRangeIndex \
import manage_addDateRangeIndex
from Products.PluginIndexes.DateRangeIndex.DateRangeIndex \
import manage_addDateRangeIndexForm
context.registerClass(DateRangeIndex,
permission='Add Pluggable Index',
constructors=(manage_addDateRangeIndexForm,
manage_addDateRangeIndex),
icon='www/index.gif',
visibility=None,
)
from Products.PluginIndexes.PathIndex.PathIndex import PathIndex
from Products.PluginIndexes.PathIndex.PathIndex \
import manage_addPathIndex
from Products.PluginIndexes.PathIndex.PathIndex \
import manage_addPathIndexForm
context.registerClass(PathIndex,
permission='Add Pluggable Index',
constructors=(manage_addPathIndexForm,
manage_addPathIndex),
icon='www/index.gif',
visibility=None,
)
from Products.PluginIndexes.BooleanIndex.BooleanIndex import BooleanIndex
from Products.PluginIndexes.BooleanIndex.BooleanIndex import \
manage_addBooleanIndex
from Products.PluginIndexes.BooleanIndex.BooleanIndex import \
manage_addBooleanIndexForm
context.registerClass(BooleanIndex,
permission='Add Pluggable Index',
constructors=(manage_addBooleanIndexForm,
manage_addBooleanIndex),
icon='www/index.gif',
visibility=None,
)
src/Products/PluginIndexes/common/ResultList.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
from BTrees.IIBTree import difference
from BTrees.IIBTree import IIBucket
from BTrees.IIBTree import weightedIntersection
from BTrees.IIBTree import weightedUnion
from BTrees.OOBTree import OOSet
from BTrees.OOBTree import union
class ResultList:
def __init__(self, d, words, index):
self._index = index
if type(words) is not OOSet:
words=OOSet(words)
self._words = words
if (type(d) is tuple):
d = IIBucket((d,))
elif type(d) is not IIBucket:
d = IIBucket(d)
self._dict=d
self.__getitem__=d.__getitem__
try: self.__nonzero__=d.__nonzero__
except: pass
self.get=d.get
def __nonzero__(self):
return not not self._dict
def bucket(self):
return self._dict
def keys(self):
return self._dict.keys()
def has_key(self, key):
return self._dict.has_key(key)
def items(self):
return self._dict.items()
def __and__(self, x):
return self.__class__(
weightedIntersection(self._dict, x._dict)[1],
union(self._words, x._words),
self._index,
)
def and_not(self, x):
return self.__class__(
difference(self._dict, x._dict),
self._words,
self._index,
)
def __or__(self, x):
return self.__class__(
weightedUnion(self._dict, x._dict)[1],
union(self._words, x._words),
self._index,
)
# return self.__class__(result, self._words+x._words, self._index)
def near(self, x):
result = IIBucket()
dict = self._dict
xdict = x._dict
xhas = xdict.has_key
positions = self._index.positions
for id, score in dict.items():
if not xhas(id): continue
p=(map(lambda i: (i,0), positions(id,self._words))+
map(lambda i: (i,1), positions(id,x._words)))
p.sort()
d = lp = 9999
li = None
lsrc = None
for i,src in p:
if i is not li and src is not lsrc and li is not None:
d = min(d,i-li)
li = i
lsrc = src
if d==lp: score = min(score,xdict[id]) # synonyms
else: score = (score+xdict[id])/d
result[id] = score
return self.__class__(
result, union(self._words, x._words), self._index)
src/Products/PluginIndexes/common/UnIndex.py deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/PluginIndexes/common/__init__.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
#############################################################################
# This code is duplicated here from Products/ZCatalog/Catalog.py to avoid a
# unnecessary dependency on ZCatalog.
import types
try:
from DocumentTemplate.cDocumentTemplate import safe_callable
except ImportError:
def safe_callable(ob):
# Works with ExtensionClasses and Acquisition.
if hasattr(ob, '__class__'):
if hasattr(ob, '__call__'):
return 1
else:
return isinstance(ob, types.ClassType)
else:
return callable(ob)
src/Products/PluginIndexes/common/randid.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
#############################################################################
import random
def randid(randint=random.randint, choice=random.choice, signs=(-1,1)):
return choice(signs)*randint(1,2000000000)
del random
src/Products/PluginIndexes/common/tests/__init__.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
#############################################################################
src/Products/PluginIndexes/common/tests/test_UnIndex.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
#############################################################################
""" Tests for common UnIndex features.
"""
import unittest
class UnIndexTests(unittest.TestCase):
def _getTargetClass(self):
from Products.PluginIndexes.common.UnIndex import UnIndex
return UnIndex
def _makeOne(self, *args, **kw):
return self._getTargetClass()(*args, **kw)
def _makeConflicted(self):
from ZODB.POSException import ConflictError
class Conflicted:
def __str__(self):
return 'Conflicted'
__repr__ = __str__
def __getattr__(self, id, default=object()):
raise ConflictError, 'testing'
return Conflicted()
def test_empty(self):
unindex = self._makeOne(id='empty')
self.assertEqual(unindex.indexed_attrs, ['empty'])
def test_removeForwardIndexEntry_with_ConflictError(self):
from ZODB.POSException import ConflictError
unindex = self._makeOne(id='conflicted')
unindex._index['conflicts'] = self._makeConflicted()
self.assertRaises(ConflictError, unindex.removeForwardIndexEntry,
'conflicts', 42)
def test_get_object_datum(self):
from Products.PluginIndexes.common.UnIndex import _marker
idx = self._makeOne('interesting')
dummy = object()
self.assertEquals(idx._get_object_datum(dummy, 'interesting'), _marker)
class DummyContent2(object):
interesting = 'GOT IT'
dummy = DummyContent2()
self.assertEquals(idx._get_object_datum(dummy, 'interesting'), 'GOT IT')
class DummyContent3(object):
exc = None
def interesting(self):
if self.exc:
raise self.exc
return 'GOT IT'
dummy = DummyContent3()
self.assertEquals(idx._get_object_datum(dummy, 'interesting'), 'GOT IT')
dummy.exc = AttributeError
self.assertEquals(idx._get_object_datum(dummy, 'interesting'), _marker)
dummy.exc = TypeError
self.assertEquals(idx._get_object_datum(dummy, 'interesting'), _marker)
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(UnIndexTests))
return suite
src/Products/PluginIndexes/common/tests/test_util.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2007 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Unit tests for util module.
"""
import unittest
from ZPublisher.HTTPRequest import record as Record
class parseIndexRequestTests(unittest.TestCase):
def _getTargetClass(self):
from Products.PluginIndexes.common.util import parseIndexRequest
return parseIndexRequest
def _makeOne(self, *args, **kw):
return self._getTargetClass()(*args, **kw)
def test_get_record(self):
record = Record()
record.query = 'foo'
record.level = 0
record.operator = 'and'
request = {'path': record}
parser = self._makeOne(request, 'path', ('query','level','operator'))
self.assertEqual(parser.get('keys'), ['foo'])
self.assertEqual(parser.get('level'), 0)
self.assertEqual(parser.get('operator'), 'and')
def test_get_dict(self):
request = {'path': {'query': 'foo', 'level': 0, 'operator': 'and'}}
parser = self._makeOne(request, 'path', ('query','level','operator'))
self.assertEqual(parser.get('keys'), ['foo'])
self.assertEqual(parser.get('level'), 0)
self.assertEqual(parser.get('operator'), 'and')
def test_get_string(self):
request = {'path': 'foo', 'path_level': 0, 'path_operator': 'and'}
parser = self._makeOne(request, 'path', ('query','level','operator'))
self.assertEqual(parser.get('keys'), ['foo'])
self.assertEqual(parser.get('level'), 0)
self.assertEqual(parser.get('operator'), 'and')
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(parseIndexRequestTests))
return suite
src/Products/PluginIndexes/common/util.py deleted 100644 → 0
View file @ 03a5df08
##############################################################################
#
# Copyright (c) 2002 Zope Foundation and Contributors.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""PluginIndexes utils.
"""
from types import InstanceType
from DateTime.DateTime import DateTime
class IndexRequestParseError(Exception):
pass
class parseIndexRequest:
"""
This class provides functionality to hide the internals of a request
send from the Catalog/ZCatalog to an index._apply_index() method.
The class understands the following type of parameters:
- old-style parameters where the query for an index as value inside
the request directory where the index name is the name of the key.
- dictionary-style parameters specify a query for an index as
an entry in the request dictionary where the key corresponds to the
name of the index and the key is a dictionary with the parameters
passed to the index.
Allowed keys of the parameter dictionary:
'query' - contains the query (either string, list or tuple) (required)
other parameters depend on the the index
- record-style parameters specify a query for an index as instance of the
Record class. This happens usually when parameters from a web form use
the "record" type e.g. <input type="text" name="path.query:record:string">.
All restrictions of the dictionary-style parameters apply to the record-style
parameters
"""
ParserException = IndexRequestParseError
def __init__(self, request, iid, options=[]):
""" parse a request from the ZPublisher and return a uniform
datastructure back to the _apply_index() method of the index
request -- the request dictionary send from the ZPublisher
iid -- Id of index
options -- a list of options the index is interested in
"""
self.id = iid
if not request.has_key(iid):
self.keys = None
return
param = request[iid]
keys = None
if isinstance(param, InstanceType) and not isinstance(param, DateTime):
""" query is of type record """
record = param
if not hasattr(record, 'query'):
raise self.ParserException(
"record for '%s' *must* contain a "
"'query' attribute" % self.id)
keys = record.query
if isinstance(keys, str):
keys = [keys.strip()]
for op in options:
if op == "query": continue
if hasattr(record, op):
setattr(self, op, getattr(record, op))
elif isinstance(param, dict):
""" query is a dictionary containing all parameters """
query = param.get("query", ())
if isinstance(query, (tuple, list)):
keys = query
else:
keys = [ query ]
for op in options:
if op == "query": continue
if param.has_key(op):
setattr(self, op, param[op])
else:
""" query is tuple, list, string, number, or something else """
if isinstance(param, (tuple, list)):
keys = param
else:
keys = [param]
for op in options:
field = iid + "_" + op
if request.has_key(field):
setattr(self, op, request[field])
self.keys = keys
def get(self, k, default_v=None):
if hasattr(self, k):
v = getattr(self, k)
if v != '':
return v
return default_v
src/Products/PluginIndexes/dtml/browseIndex.dtml deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/PluginIndexes/interfaces.py deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/Catalog.py deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/CatalogAwareness.py deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/CatalogBrains.py deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/CatalogPathAwareness.py deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/Lazy.py deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/ProgressHandler.py deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/ZCatalog.py deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/ZCatalogIndexes.py deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/__init__.py deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/dtml/addIndexForm.dtml deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/dtml/addZCatalog.dtml deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/dtml/catalogAdvanced.dtml deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/dtml/catalogFind.dtml deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/dtml/catalogIndexes.dtml deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/dtml/catalogPlan.dtml deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/dtml/catalogReport.dtml deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/dtml/catalogSchema.dtml deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/dtml/catalogStatus.dtml deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/dtml/catalogView.dtml deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/dtml/editCatalogerForm.dtml deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/interfaces.py deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/plan.py deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/tests/test_brains.py deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/tests/test_catalog.py deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/tests/test_lazy.py deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/tests/test_plan.py deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
src/Products/ZCatalog/tests/test_zcatalog.py deleted 100644 → 0
View file @ 03a5df08
This diff is collapsed.
versions.cfg
View file @ 97bb6b12
...@@ -22,6 +22,7 @@ Products.MIMETools = 2.13.0 ...@@ -22,6 +22,7 @@ Products.MIMETools = 2.13.0
Products.OFSP = 2.13.2 Products.OFSP = 2.13.2
Products.PythonScripts = 2.13.0 Products.PythonScripts = 2.13.0
Products.StandardCacheManagers = 2.13.0 Products.StandardCacheManagers = 2.13.0
Products.ZCatalog = 2.13.0
Products.ZCTextIndex = 2.13.1 Products.ZCTextIndex = 2.13.1
Record = 2.13.0 Record = 2.13.0
tempstorage = 2.12.1 tempstorage = 2.12.1
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7