Commit 24560732 authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki

patch Products.BTreeFolder2 to drop _mt_index.

drop _mt_index in BTreeFolder2 that is a main cause of ConflictError
in parallel processing.
filter by meta_type in objectIds() and objectValues() are still
supported, but slow.
parent e770e9f7
diff -ur Products.BTreeFolder2-2.13.5.orig/src/Products/BTreeFolder2/BTreeFolder2.py Products.BTreeFolder2-2.13.5/src/Products/BTreeFolder2/BTreeFolder2.py
--- Products.BTreeFolder2-2.13.5.orig/src/Products/BTreeFolder2/BTreeFolder2.py 2015-06-19 05:59:00.000000000 +0900
+++ Products.BTreeFolder2-2.13.5/src/Products/BTreeFolder2/BTreeFolder2.py 2016-07-19 16:13:30.274201719 +0900
@@ -28,8 +28,6 @@
from Acquisition import aq_base
from App.special_dtml import DTMLFile
from BTrees.Length import Length
-from BTrees.OIBTree import OIBTree
-from BTrees.OIBTree import union
from BTrees.OOBTree import OOBTree
from OFS.event import ObjectWillBeAddedEvent
from OFS.event import ObjectWillBeRemovedEvent
@@ -94,7 +92,6 @@
_tree = None # OOBTree: { id -> object }
_count = None # A BTrees.Length
_v_nextid = 0 # The integer component of the next generated ID
- _mt_index = None # OOBTree: { meta_type -> OIBTree: { id -> 1 } }
title = ''
# superValues() looks for the _objects attribute, but the implementation
@@ -109,7 +106,6 @@
def _initBTrees(self):
self._tree = OOBTree()
self._count = Length()
- self._mt_index = OOBTree()
def _populateFromFolder(self, source):
"""Fill this folder with the contents of another folder.
@@ -174,20 +170,6 @@
if key not in self._tree:
raise AssertionError(
"Missing value for key: %s" % repr(key))
- check(self._mt_index)
- keys = set(self._tree.keys())
- for key, value in self._mt_index.items():
- if (key not in self._mt_index
- or self._mt_index[key] is not value):
- raise AssertionError(
- "Missing or incorrect meta_type index: %s"
- % repr(key))
- check(value)
- for k in value.keys():
- if k not in value or k not in keys:
- raise AssertionError(
- "Missing values for meta_type index: %s"
- % repr(key))
return 1
except AssertionError:
LOG.warn('Detected damage to %s. Fixing now.' % path,
@@ -195,13 +177,6 @@
try:
self._tree = OOBTree(self._tree)
keys = set(self._tree.keys())
- mt_index = OOBTree()
- for key, value in self._mt_index.items():
- for name in tuple(value.keys()):
- if name not in keys:
- del value[name]
- mt_index[key] = OIBTree(value)
- self._mt_index = mt_index
new = len(keys)
if self._count() != new:
self._count.set(new)
@@ -249,33 +224,13 @@
raise KeyError('There is already an item named "%s".' % id)
tree[id] = object
self._count.change(1)
- # Update the meta type index.
- mti = self._mt_index
- meta_type = getattr(object, 'meta_type', None)
- if meta_type is not None:
- ids = mti.get(meta_type, None)
- if ids is None:
- ids = OIBTree()
- mti[meta_type] = ids
- ids[id] = 1
def _delOb(self, id):
"""Remove the named object from the folder.
"""
tree = self._tree
- meta_type = getattr(tree[id], 'meta_type', None)
del tree[id]
self._count.change(-1)
- # Update the meta type index.
- if meta_type is not None:
- mti = self._mt_index
- ids = mti.get(meta_type, None)
- if ids is not None and id in ids:
- del ids[id]
- if not ids:
- # Removed the last object of this meta_type.
- # Prune the index.
- del mti[meta_type]
security.declareProtected(view_management_screens, 'getBatchObjectListing')
def getBatchObjectListing(self, REQUEST=None):
@@ -364,16 +319,14 @@
if isinstance(spec, str):
spec = [spec]
- set = None
- mti = self._mt_index
- for meta_type in spec:
- ids = mti.get(meta_type, None)
- if ids is not None:
- set = union(set, ids)
- if set is None:
- return ()
- else:
- return set.keys()
+ # meta_type filter by 'spec' is still supported but slow.
+ ids = []
+ for id in self._tree.keys():
+ obj = self._tree[id]
+ meta_type = getattr(obj, 'meta_type', None)
+ if meta_type is not None and meta_type in spec:
+ ids.append(id)
+ return tuple(ids)
def __contains__(self, name):
return name in self._tree
diff -ur Products.BTreeFolder2-2.13.5.orig/src/Products/BTreeFolder2/tests/testBTreeFolder2.py Products.BTreeFolder2-2.13.5/src/Products/BTreeFolder2/tests/testBTreeFolder2.py
--- Products.BTreeFolder2-2.13.5.orig/src/Products/BTreeFolder2/tests/testBTreeFolder2.py 2015-06-19 05:59:00.000000000 +0900
+++ Products.BTreeFolder2-2.13.5/src/Products/BTreeFolder2/tests/testBTreeFolder2.py 2016-07-19 15:52:43.712322217 +0900
@@ -255,11 +255,6 @@
# Now it's fixed.
self.assert_(self.f._cleanup())
- from BTrees.OIBTree import OIBTree
- tree = self.f._mt_index['d'] = OIBTree()
- tree['e'] = 1
- self.assert_(not self.f._cleanup())
-
# Verify the management interface also works,
# but don't test return values.
self.f.manage_cleanup()
...@@ -467,6 +467,7 @@ initialization = ...@@ -467,6 +467,7 @@ initialization =
[eggs] [eggs]
<= neoppod <= neoppod
eggs = eggs =
# Install customised eggs
${numpy:egg} ${numpy:egg}
${matplotlib:egg} ${matplotlib:egg}
${python-mysqlclient:egg} ${python-mysqlclient:egg}
...@@ -477,6 +478,13 @@ eggs = ...@@ -477,6 +478,13 @@ eggs =
${pysvn-python:egg} ${pysvn-python:egg}
${pycrypto-python:egg} ${pycrypto-python:egg}
${scikit-learn:egg} ${scikit-learn:egg}
# Install patched eggs
Acquisition
Products.BTreeFolder2
Products.DCWorkflow
python-magic
lock_file lock_file
astor astor
PyStemmer PyStemmer
...@@ -498,7 +506,6 @@ eggs = ...@@ -498,7 +506,6 @@ eggs =
ply ply
pyflakes pyflakes
pypdf2 pypdf2
python-magic
python-memcached python-memcached
pytz pytz
requests requests
...@@ -528,8 +535,6 @@ eggs = ...@@ -528,8 +535,6 @@ eggs =
# Zope # Zope
ZODB3 ZODB3
Zope2 Zope2
# Zope acquisition patch
Acquisition
# Other Zope 2 packages # Other Zope 2 packages
Products.PluggableAuthService Products.PluggableAuthService
...@@ -542,7 +547,6 @@ eggs = ...@@ -542,7 +547,6 @@ eggs =
Products.CMFDefault Products.CMFDefault
Products.CMFTopic Products.CMFTopic
Products.CMFUid Products.CMFUid
Products.DCWorkflow
Products.GenericSetup Products.GenericSetup
five.localsitemanager five.localsitemanager
...@@ -584,6 +588,9 @@ extra-paths = ...@@ -584,6 +588,9 @@ extra-paths =
patch-binary = ${patch:location}/bin/patch patch-binary = ${patch:location}/bin/patch
Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic.patch#e8029103350dad364d25747514a20327 Acquisition-patches = ${:_profile_base_location_}/../../component/egg-patch/Acquisition/aq_dynamic.patch#e8029103350dad364d25747514a20327
Acquisition-patch-options = -p1 Acquisition-patch-options = -p1
Products.BTreeFolder2-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.BTreeFolder2/drop_mt_index.patch#ff664ea8113d1370635bb6bbcea6406e
Products.BTreeFolder2-patch-options = -p1
Products.BTreeFolder2-patch-revision = 2
Products.DCWorkflow-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.DCWorkflow/workflow_method.patch#975b49e96bae33ac8563454fe5fa9899 Products.DCWorkflow-patches = ${:_profile_base_location_}/../../component/egg-patch/Products.DCWorkflow/workflow_method.patch#975b49e96bae33ac8563454fe5fa9899
Products.DCWorkflow-patch-options = -p1 Products.DCWorkflow-patch-options = -p1
python-magic-patches = ${:_profile_base_location_}/../../component/egg-patch/python_magic/magic.patch#de0839bffac17801e39b60873a6c2068 python-magic-patches = ${:_profile_base_location_}/../../component/egg-patch/python_magic/magic.patch#de0839bffac17801e39b60873a6c2068
...@@ -621,6 +628,7 @@ scripts += ...@@ -621,6 +628,7 @@ scripts +=
# patched eggs # patched eggs
Acquisition = 2.13.9+SlapOSPatched001 Acquisition = 2.13.9+SlapOSPatched001
Products.DCWorkflow = 2.2.4+SlapOSPatched001 Products.DCWorkflow = 2.2.4+SlapOSPatched001
Products.BTreeFolder2 = 2.13.5+SlapOSPatched002
pysvn = 1.7.10+SlapOSPatched002 pysvn = 1.7.10+SlapOSPatched002
python-ldap = 2.4.25+SlapOSPatched001 python-ldap = 2.4.25+SlapOSPatched001
python-magic = 0.4.11+SlapOSPatched001 python-magic = 0.4.11+SlapOSPatched001
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment