Remove dependency on Products.ZCatalog and Record, closes #38.

CHANGES.rst

@@ -221,4 +221,6 @@ Restructuring
     - ``Products.PythonScripts``
     - ``Products.SiteErrorLog``
     - ``Products.StandardCacheManagers``
+    - ``Products.ZCatalog``
     - ``Products.ZCTextIndex``
+    - ``Record``

buildout.cfg

@@ -19,6 +19,7 @@ parts =
 sources-dir = develop
 auto-checkout =
     Products.SiteErrorLog
+    Products.ZCatalog
 
 
 [test]
@@ -55,8 +56,6 @@ eggs =
     MultiMapping
     Persistence
     Products.OFSP
-    Products.ZCatalog
-    Record
     Zope2
 # RestrictedPython has an optional dependency on DateTime, make sure to run its
 # tests with DateTime being available
@@ -72,7 +71,9 @@ eggs =
     Products.PythonScripts
     Products.SiteErrorLog
     Products.StandardCacheManagers
+    Products.ZCatalog
     Products.ZCTextIndex
+    Record
 
 
 [ztktests]

setup.py

@@ -60,8 +60,6 @@ setup(
         'PasteDeploy',
         'Persistence',
         'Products.OFSP >= 2.13.2',
-        'Products.ZCatalog',
-        'Record',
         'RestrictedPython',
         'ZConfig >= 2.9.2',
         'ZODB',

src/OFS/deprecated.zcml

@@ -15,9 +15,4 @@
   <five:deprecatedManageAddDelete
       class="Products.SiteAccess.VirtualHostMonster.VirtualHostMonster"/>
 
-  <five:deprecatedManageAddDelete
-      class="Products.ZCatalog.CatalogAwareness.CatalogAware"/>
-  <five:deprecatedManageAddDelete
-      class="Products.ZCatalog.CatalogPathAwareness.CatalogAware"/>
-
 </configure>

src/ZTUtils/Lazy.py

+##############################################################################
+#
+# Copyright (c) 2002 Zope Foundation and Contributors.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+from itertools import islice, count
+
+_marker = object()
+
+
+class Lazy(object):
+
+    # Allow (reluctantly) access to unprotected attributes
+    __allow_access_to_unprotected_subobjects__ = True
+    _len = _marker
+    _rlen = _marker
+
+    @property
+    def actual_result_count(self):
+        if self._rlen is not _marker:
+            return self._rlen
+        self._rlen = len(self)
+        return self._rlen
+
+    @actual_result_count.setter
+    def actual_result_count(self, value):
+        self._rlen = value
+
+    def __repr__(self):
+        return repr(list(self))
+
+    def __len__(self):
+        # This is a worst-case len, subclasses should try to do better
+        if self._len is not _marker:
+            return self._len
+        l = len(self._data)
+        while 1:
+            try:
+                self[l]
+                l = l + 1
+            except Exception:
+                self._len = l
+                return l
+
+    def __add__(self, other):
+        if not isinstance(other, Lazy):
+            raise TypeError(
+                "Can not concatenate objects. Both must be lazy sequences.")
+        return LazyCat([self, other])
+
+    def __getslice__(self, i1, i2):
+        r = []
+        for i in islice(count(i1), i2 - i1):
+            try:
+                r.append(self[i])
+            except IndexError:
+                return r
+        return r
+
+    slice = __getslice__
+
+
+class LazyCat(Lazy):
+    """Lazy concatenation of one or more sequences. Should be handy
+    for accessing small parts of big searches.
+    """
+
+    def __init__(self, sequences, length=None, actual_result_count=None):
+        flattened_count = 0
+        if len(sequences) < 100:
+            # Optimize structure of LazyCats to avoid nesting
+            # We don't do this for large numbers of input sequences
+            # to make instantiation faster instead
+            flattened_seq = []
+            for s in sequences:
+                if isinstance(s, LazyCat):
+                    # If one of the sequences passed is itself a LazyCat, add
+                    # its base sequences rather than nest LazyCats
+                    if getattr(s, '_seq', None) is None:
+                        flattened_seq.extend([s._data])
+                    else:
+                        flattened_seq.extend(s._seq)
+                    flattened_count += s.actual_result_count
+                elif isinstance(s, Lazy):
+                    flattened_seq.append(s)
+                    flattened_count += s.actual_result_count
+                else:
+                    flattened_seq.append(s)
+                    flattened_count += len(s)
+            sequences = flattened_seq
+        self._seq = sequences
+        self._data = []
+        self._sindex = 0
+        self._eindex = -1
+        if length is not None:
+            self._len = length
+        if actual_result_count is not None:
+            self.actual_result_count = actual_result_count
+        else:
+            self.actual_result_count = flattened_count
+
+    def __getitem__(self, index):
+        data = self._data
+        try:
+            seq = self._seq
+        except AttributeError:
+            return data[index]
+
+        i = index
+        if i < 0:
+            i = len(self) + i
+        if i < 0:
+            raise IndexError(index)
+
+        ind = len(data)
+        if i < ind:
+            return data[i]
+        ind = ind - 1
+
+        sindex = self._sindex
+        try:
+            s = seq[sindex]
+        except Exception:
+            raise IndexError(index)
+        eindex = self._eindex
+        while i > ind:
+            try:
+                eindex = eindex + 1
+                v = s[eindex]
+                data.append(v)
+                ind = ind + 1
+            except IndexError:
+                self._sindex = sindex = sindex + 1
+                try:
+                    s = self._seq[sindex]
+                except Exception:
+                    del self._seq
+                    del self._sindex
+                    del self._eindex
+                    raise IndexError(index)
+                self._eindex = eindex = -1
+        self._eindex = eindex
+        return data[i]
+
+    def __len__(self):
+        # Make len of LazyCat only as expensive as the lens
+        # of its underlying sequences
+        if self._len is not _marker:
+            return self._len
+        l = 0
+        try:
+            for s in self._seq:
+                l += len(s)
+        except AttributeError:
+            l = len(self._data)
+        self._len = l
+        return l
+
+
+class LazyMap(Lazy):
+    """Act like a sequence, but get data from a filtering process.
+    Don't access data until necessary
+    """
+
+    def __init__(self, func, seq, length=None, actual_result_count=None):
+        self._seq = seq
+        self._data = {}
+        self._func = func
+        if length is not None:
+            self._len = length
+        else:
+            self._len = len(seq)
+        if actual_result_count is not None:
+            self.actual_result_count = actual_result_count
+        else:
+            self.actual_result_count = self._len
+
+    def __getitem__(self, index):
+        data = self._data
+        if index in data:
+            return data[index]
+        value = data[index] = self._func(self._seq[index])
+        return value
+
+
+class LazyFilter(Lazy):
+    """Act like a sequence, but get data from a filtering process.
+    Don't access data until necessary. Only data for which test(data)
+    returns true will be considered part of the set.
+    """
+
+    def __init__(self, test, seq):
+        self._seq = seq
+        self._data = []
+        self._eindex = -1
+        self._test = test
+
+    def __getitem__(self, index):
+        data = self._data
+        try:
+            s = self._seq
+        except AttributeError:
+            return data[index]
+
+        i = index
+        if i < 0:
+            i = len(self) + i
+        if i < 0:
+            raise IndexError(index)
+
+        ind = len(data)
+        if i < ind:
+            return data[i]
+        ind = ind - 1
+
+        test = self._test
+        e = self._eindex
+        while i > ind:
+            try:
+                e = e + 1
+                v = s[e]
+                if test(v):
+                    data.append(v)
+                    ind = ind + 1
+            except IndexError:
+                del self._test
+                del self._seq
+                del self._eindex
+                raise IndexError(index)
+        self._eindex = e
+        return data[i]
+
+
+class LazyMop(Lazy):
+    """Act like a sequence, but get data from a filtering process.
+    Don't access data until necessary. If the filter raises an exception
+    for a given item, then that item isn't included in the sequence.
+    """
+
+    def __init__(self, test, seq):
+        self._seq = seq
+        self._data = []
+        self._eindex = -1
+        self._test = test
+
+    def __getitem__(self, index):
+        data = self._data
+        try:
+            s = self._seq
+        except AttributeError:
+            return data[index]
+
+        i = index
+        if i < 0:
+            i = len(self) + i
+        if i < 0:
+            raise IndexError(index)
+
+        ind = len(data)
+        if i < ind:
+            return data[i]
+        ind = ind - 1
+
+        test = self._test
+        e = self._eindex
+        while i > ind:
+            try:
+                e = e + 1
+                v = s[e]
+                try:
+                    v = test(v)
+                    data.append(v)
+                    ind = ind + 1
+                except Exception:
+                    pass
+            except IndexError:
+                del self._test
+                del self._seq
+                del self._eindex
+                raise IndexError(index)
+        self._eindex = e
+        return data[i]
+
+
+class LazyValues(Lazy):
+    """Given a sequence of two tuples typically (key, value) act as
+    though we are just a list of the values lazily"""
+
+    def __init__(self, seq):
+        self._seq = seq
+
+    def __len__(self):
+        if self._len is not _marker:
+            return self._len
+        self._len = len(self._seq)
+        return self._len
+
+    def __getitem__(self, index):
+        return self._seq[index][1]
+
+    def __getslice__(self, start, end):
+        return self.__class__(self._seq[start:end])
+
+    slice = __getslice__

src/ZTUtils/Zope.py

@@ -19,9 +19,9 @@ import urllib
 
 from AccessControl import getSecurityManager
 from DateTime.DateTime import DateTime
-from Products.ZCatalog.Lazy import Lazy
 
 from ZTUtils.Batch import Batch
+from ZTUtils.Lazy import Lazy
 from ZTUtils.SimpleTree import SimpleTreeMaker
 from ZTUtils.Tree import decodeExpansion
 from ZTUtils.Tree import encodeExpansion

src/ZTUtils/tests/testLazy.py

+##############################################################################
+#
+# Copyright (c) 2002 Zope Foundation and Contributors.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+import unittest
+
+
+class BaseSequenceTest(object):
+
+    def _compare(self, lseq, seq):
+        self.assertEqual(len(lseq), len(seq))
+        self.assertEqual(list(lseq), seq)
+
+    def test_actual_result_count(self):
+        lcat = self._createLSeq(range(10))
+        self.assertEqual(len(lcat), 10)
+        self.assertEqual(lcat.actual_result_count, 10)
+
+        lcat.actual_result_count = 20
+        self.assertEqual(len(lcat), 10)
+        self.assertEqual(lcat.actual_result_count, 20)
+
+
+class TestLazyCat(unittest.TestCase, BaseSequenceTest):
+
+    def _createLSeq(self, *sequences):
+        from ZTUtils.Lazy import LazyCat
+        return LazyCat(sequences)
+
+    def _createLValues(self, seq):
+        from ZTUtils.Lazy import LazyValues
+        return LazyValues(seq)
+
+    def test_empty(self):
+        lcat = self._createLSeq([])
+        self._compare(lcat, [])
+        self.assertEqual(lcat.actual_result_count, 0)
+
+    def test_repr(self):
+        lcat = self._createLSeq([0, 1])
+        self.assertEquals(repr(lcat), repr([0, 1]))
+
+    def test_init_single(self):
+        seq = range(10)
+        lcat = self._createLSeq(seq)
+        self._compare(lcat, seq)
+        self.assertEqual(lcat.actual_result_count, 10)
+
+    def test_add(self):
+        seq1 = range(10)
+        seq2 = range(10, 20)
+        lcat1 = self._createLSeq(seq1)
+        lcat2 = self._createLSeq(seq2)
+        lcat = lcat1 + lcat2
+        self._compare(lcat, range(20))
+        self.assertEqual(lcat.actual_result_count, 20)
+
+    def test_add_after_getitem(self):
+        seq1 = range(10)
+        seq2 = range(10, 20)
+        lcat1 = self._createLSeq(seq1)
+        lcat2 = self._createLSeq(seq2)
+        # turning lcat1 into a list will flatten it into _data and remove _seq
+        list(lcat1)
+        lcat = lcat1 + lcat2
+        self._compare(lcat, range(20))
+        self.assertEqual(lcat.actual_result_count, 20)
+
+    def test_init_multiple(self):
+        from string import hexdigits, letters
+        seq1 = range(10)
+        seq2 = list(hexdigits)
+        seq3 = list(letters)
+        lcat = self._createLSeq(seq1, seq2, seq3)
+        self._compare(lcat, seq1 + seq2 + seq3)
+
+    def test_init_nested(self):
+        from string import hexdigits, letters
+        seq1 = range(10)
+        seq2 = list(hexdigits)
+        seq3 = list(letters)
+        lcat = self._createLSeq(
+            *[self._createLSeq(seq) for seq in (seq1, seq2, seq3)])
+        self._compare(lcat, seq1 + seq2 + seq3)
+
+    def test_slicing(self):
+        from string import hexdigits, letters
+        seq1 = range(10)
+        seq2 = list(hexdigits)
+        seq3 = list(letters)
+        lcat = self._createLSeq(
+            *[self._createLSeq(seq) for seq in (seq1, seq2, seq3)])
+        self._compare(lcat[5:-5], seq1[5:] + seq2 + seq3[:-5])
+
+    def test_length(self):
+        # Unaccessed length
+        lcat = self._createLSeq(range(10))
+        self.assertEqual(len(lcat), 10)
+        self.assertEqual(lcat.actual_result_count, 10)
+
+        # Accessed in the middle
+        lcat = self._createLSeq(range(10))
+        lcat[4]
+        self.assertEqual(len(lcat), 10)
+        self.assertEqual(lcat.actual_result_count, 10)
+
+        # Accessed after the lcat is accessed over the whole range
+        lcat = self._createLSeq(range(10))
+        lcat[:]
+        self.assertEqual(len(lcat), 10)
+        self.assertEqual(lcat.actual_result_count, 10)
+
+    def test_actual_result_count(self):
+        # specify up-front
+        lcat = self._createLSeq(range(10))
+        lcat.actual_result_count = 100
+
+        self.assertEqual(len(lcat), 10)
+        self.assertEqual(lcat.actual_result_count, 100)
+
+        lvalues = self._createLValues([])
+        self.assertEqual(len(lvalues), 0)
+        self.assertEqual(lvalues.actual_result_count, 0)
+
+        combined = lvalues + lcat
+        self.assertEqual(len(combined), 10)
+        self.assertEqual(combined.actual_result_count, 100)
+
+        combined.actual_result_count = 5
+        self.assertEqual(combined.actual_result_count, 5)
+
+
+class TestLazyMap(TestLazyCat):
+
+    def _createLSeq(self, *seq):
+        return self._createLMap(lambda x: x, *seq)
+
+    def _createLMap(self, mapfunc, *seq):
+        from ZTUtils.Lazy import LazyMap
+        totalseq = []
+        for s in seq:
+            totalseq.extend(s)
+        return LazyMap(mapfunc, totalseq)
+
+    def test_map(self):
+        from string import hexdigits, letters
+        seq1 = range(10)
+        seq2 = list(hexdigits)
+        seq3 = list(letters)
+
+        def to_lower(x):
+            return str(x).lower()
+
+        lmap = self._createLMap(to_lower, seq1, seq2, seq3)
+        self._compare(lmap, [str(x).lower() for x in (seq1 + seq2 + seq3)])
+
+    def testMapFuncIsOnlyCalledAsNecessary(self):
+        seq = range(10)
+        count = [0]  # closure only works with list, and `nonlocal` in py3
+
+        def func(x):
+            count[0] += 1
+            return x
+
+        lmap = self._createLMap(func, seq)
+        self.assertEqual(lmap[5], 5)
+        self.assertEqual(count[0], 1)
+
+
+class TestLazyFilter(TestLazyCat):
+
+    def _createLSeq(self, *seq):
+        return self._createLFilter(lambda x: True, *seq)
+
+    def _createLFilter(self, filter, *seq):
+        from ZTUtils.Lazy import LazyFilter
+        totalseq = []
+        for s in seq:
+            totalseq.extend(s)
+        return LazyFilter(filter, totalseq)
+
+    def test_filter(self):
+        from string import hexdigits, letters
+        seq1 = range(10)
+        seq2 = list(hexdigits)
+        seq3 = list(letters)
+
+        def is_alpha(x):
+            return str(x).isalpha()
+
+        lmap = self._createLFilter(is_alpha, seq1, seq2, seq3)
+        self._compare(lmap, seq2[10:] + seq3)
+
+    def test_length_with_filter(self):
+        from string import letters
+        lower_length = len([x for x in letters if x.islower()])
+
+        # Unaccessed length
+        lfilter = self._createLFilter(lambda x: x.islower(), list(letters))
+        self.assertEqual(len(lfilter), lower_length)
+
+        # Accessed in the middle
+        lfilter = self._createLFilter(lambda x: x.islower(), list(letters))
+        lfilter[13]
+        self.assertEqual(len(lfilter), lower_length)
+
+        # Accessed after the lcat is accessed over the whole range
+        lfilter = self._createLFilter(lambda x: x.islower(), list(letters))
+        lfilter[:]
+        self.assertEqual(len(lfilter), lower_length)
+
+
+class TestLazyMop(TestLazyCat):
+
+    def _createLSeq(self, *seq):
+        return self._createLMop(lambda x: x, *seq)
+
+    def _createLMop(self, mapfunc, *seq):
+        from ZTUtils.Lazy import LazyMop
+        totalseq = []
+        for s in seq:
+            totalseq.extend(s)
+        return LazyMop(mapfunc, totalseq)
+
+    def test_mop(self):
+        from string import hexdigits, letters
+        seq1 = range(10)
+        seq2 = list(hexdigits)
+        seq3 = list(letters)
+
+        def filter(x):
+            if isinstance(x, int):
+                raise ValueError
+            return x.lower()
+
+        lmop = self._createLMop(filter, seq1, seq2, seq3)
+        self._compare(lmop, [str(x).lower() for x in (seq2 + seq3)])
+
+    def test_length_with_filter(self):
+        from string import letters
+        letter_length = len(letters)
+        seq = range(10) + list(letters)
+
+        def filter(x):
+            if isinstance(x, int):
+                raise ValueError
+            return x.lower()
+
+        # Unaccessed length
+        lmop = self._createLMop(filter, seq)
+        self.assertEqual(len(lmop), letter_length)
+
+        # Accessed in the middle
+        lmop = self._createLMop(filter, seq)
+        lmop[26]
+        self.assertEqual(len(lmop), letter_length)
+
+        # Accessed after the lcat is accessed over the whole range
+        lmop = self._createLMop(filter, letters)
+        lmop[:]
+        self.assertEqual(len(lmop), letter_length)
+
+
+class TestLazyValues(unittest.TestCase, BaseSequenceTest):
+
+    def _createLSeq(self, seq):
+        from ZTUtils.Lazy import LazyValues
+        return LazyValues(seq)
+
+    def test_empty(self):
+        lvals = self._createLSeq([])
+        self._compare(lvals, [])
+
+    def test_values(self):
+        from string import letters
+        seq = zip(letters, range(10))
+        lvals = self._createLSeq(seq)
+        self._compare(lvals, range(10))
+
+    def test_slicing(self):
+        from string import letters
+        seq = zip(letters, range(10))
+        lvals = self._createLSeq(seq)
+        self._compare(lvals[2:-2], range(2, 8))