Commit 11456c69 authored by Arnaud Fontaine's avatar Arnaud Fontaine

ZODB Components: Fix deadlock between import lock and aq_method_lock.

This only happens when using ZEO (see source code comments).

Steps to reproduce:
  1. Edit a ZODB Component in one tab.
  2. At the same time, run Unit Tests in another tab.

Backtrace:

  # ThreadID: 140153540167424
  File: "eggs/Zope2-2.13.22-py2.7.egg/ZServer/PubCore/ZServerPublisher.py", line 31, in __init__
    response=b)
  File: "eggs/Zope2-2.13.22-py2.7.egg/ZPublisher/Publish.py", line 455, in publish_module
    environ, debug, request, response)
  File: "eggs/Zope2-2.13.22-py2.7.egg/ZPublisher/Publish.py", line 249, in publish_module_standard
    response = publish(request, module_name, after_list, debug=debug)
  File: "parts/erp5/Products/Localizer/patches.py", line 84, in new_publish
    x = zope_publish(request, module_name, after_list, debug)
  File: "eggs/Zope2-2.13.22-py2.7.egg/ZPublisher/Publish.py", line 138, in publish
    request, bind=1)
  File: "eggs/Zope2-2.13.22-py2.7.egg/ZPublisher/mapply.py", line 44, in mapply
    f, count = zope.publisher.publish.unwrapMethod(object)
  File: "eggs/zope.publisher-3.12.6-py2.7.egg/zope/publisher/publish.py", line 46, in unwrapMethod
    elif getattr(unwrapped, 'func_code', None) is not None:
  File: "eggs/Products.ExternalMethod-2.13.0-py2.7.egg/Products/ExternalMethod/ExternalMethod.py", line 106, in <lambda>
    func_code = ComputedAttribute(lambda self: self.getFuncCode())
  File: "eggs/Products.ExternalMethod-2.13.0-py2.7.egg/Products/ExternalMethod/ExternalMethod.py", line 190, in getFuncCode
    self._v_f = self.getFunction()
  File: "parts/erp5/Products/ERP5Type/patches/ExternalMethod.py", line 29, in getFunction
    level=0)
  File: "parts/erp5/Products/ERP5Type/dynamic/component_package.py", line 407, in load_module
    with aq_method_lock:
  File: "parts/python2.7/lib/python2.7/threading.py", line 174, in acquire
    rc = self.__block.acquire(blocking)

  => 1. Acquire Import lock in getFunction() (ZODB Component import)
     2. Try to acquire aq_method_lock

  # ThreadID: 140153468495616
  File: "eggs/Zope2-2.13.22-py2.7.egg/ZServer/PubCore/ZServerPublisher.py", line 31, in __init__
    response=b)
  File: "eggs/Zope2-2.13.22-py2.7.egg/ZPublisher/Publish.py", line 455, in publish_module
    environ, debug, request, response)
  File: "eggs/Zope2-2.13.22-py2.7.egg/ZPublisher/Publish.py", line 249, in publish_module_standard
    response = publish(request, module_name, after_list, debug=debug)
  File: "parts/erp5/Products/Localizer/patches.py", line 84, in new_publish
    x = zope_publish(request, module_name, after_list, debug)
  File: "eggs/Zope2-2.13.22-py2.7.egg/ZPublisher/Publish.py", line 138, in publish
    request, bind=1)
  File: "eggs/Zope2-2.13.22-py2.7.egg/ZPublisher/mapply.py", line 77, in mapply
    if debug is not None: return debug(object,args,context)
  File: "eggs/Zope2-2.13.22-py2.7.egg/ZPublisher/Publish.py", line 48, in call_object
    result=apply(object,args) # Type s<cr> to step into published object.
  File: "parts/erp5/Products/TimerService/TimerService.py", line 90, in process_timer
    subscriptions.append(self.unrestrictedTraverse(path))
  File: "eggs/Zope2-2.13.22-py2.7.egg/OFS/Traversable.py", line 249, in unrestrictedTraverse
    if getattr(aq_base(obj), name, _marker) is not _marker:
  File: "parts/erp5/Products/ERP5Type/dynamic/lazy_class.py", line 107, in __getattribute__
    self.__class__.loadClass()
  File: "parts/erp5/Products/ERP5Type/dynamic/lazy_class.py", line 326, in loadClass
    class_definition = generatePortalTypeClass(site, portal_type)
  File: "parts/erp5/Products/ERP5Type/dynamic/portal_type_class.py", line 143, in generatePortalTypeClass
    type_class = portal_type.getTypeClass()
  File: "eggs/ZODB3-3.10.5+slapospatched001-py2.7-linux-x86_64.egg/ZODB/Connection.py", line 860, in setstate
    self._setstate(obj)
  File: "eggs/ZODB3-3.10.5+slapospatched001-py2.7-linux-x86_64.egg/ZODB/Connection.py", line 914, in _setstate
    self._reader.setGhostState(obj, p)
  File: "eggs/ZODB3-3.10.5+slapospatched001-py2.7-linux-x86_64.egg/ZODB/serialize.py", line 612, in setGhostState
    state = self.getState(pickle)
  File: "eggs/ZODB3-3.10.5+slapospatched001-py2.7-linux-x86_64.egg/ZODB/serialize.py", line 604, in getState
    unpickler.load() # skip the class metadata
  File: "eggs/ZODB3-3.10.5+slapospatched001-py2.7-linux-x86_64.egg/ZODB/serialize.py", line 474, in find_global
    return factory(conn, modulename, name)
  File: "eggs/Zope2-2.13.22-py2.7.egg/Zope2/App/ClassFactory.py", line 21, in ClassFactory
    m=__import__(module, _globals, _globals, _silly)

  => 1. Acquire aq_method_lock (generatePortalTypeClass())
     2. Try to import module and acquire Import lock
parent 22fea7d8
...@@ -241,102 +241,76 @@ class ComponentDynamicPackage(ModuleType): ...@@ -241,102 +241,76 @@ class ComponentDynamicPackage(ModuleType):
As per PEP-302, raise an ImportError if the Loader could not load the As per PEP-302, raise an ImportError if the Loader could not load the
module for any reason... module for any reason...
""" """
# In Python < 3.3, the import lock is a global lock for all modules: site = getSite()
# http://bugs.python.org/issue9260 name = fullname[len(self._namespace_prefix):]
#
# So, release the import lock acquired by import statement on all hooks to
# load objects from ZODB. When an object is requested from ZEO, it sends a
# RPC request and lets the asyncore thread gets the reply. This reply may
# be a tuple (PICKLE, TID), sent directly to the first thread, or an
# Exception, which tries to import a ZODB module and thus creates a
# deadlock because of the global import lock
#
# Also, handle the case where find_module() may be called without import
# statement as it does change anything in sys.modules
import_lock_held = True
try:
imp.release_lock()
except RuntimeError:
import_lock_held = False
try: # if only Version package (erp5.component.XXX.VERSION_version) is
site = getSite() # requested to be loaded, then create it if necessary
name = fullname[len(self._namespace_prefix):] if name.endswith('_version'):
version = name[:-self.__version_suffix_len]
return (version in site.getVersionPriorityNameList() and
self._getVersionPackage(version) or None)
# if only Version package (erp5.component.XXX.VERSION_version) is module_fullname_alias = None
# requested to be loaded, then create it if necessary version_package_name = name[:-self.__version_suffix_len]
if name.endswith('_version'):
version = name[:-self.__version_suffix_len]
return (version in site.getVersionPriorityNameList() and
self._getVersionPackage(version) or None)
module_fullname_alias = None # If a specific version of the Component has been requested
version_package_name = name[:-self.__version_suffix_len] if '.' in name:
try:
# If a specific version of the Component has been requested version, name = name.split('.')
if '.' in name: version = version[:-self.__version_suffix_len]
try: except ValueError, error:
version, name = name.split('.') raise ImportError("%s: should be %s.VERSION.COMPONENT_REFERENCE (%s)" % \
version = version[:-self.__version_suffix_len] (fullname, self._namespace, error))
except ValueError, error:
raise ImportError("%s: should be %s.VERSION.COMPONENT_REFERENCE (%s)" % \
(fullname, self._namespace, error))
try: try:
component_id = self._registry_dict[name][version][0] component_id = self._registry_dict[name][version][0]
except KeyError: except KeyError:
raise ImportError("%s: version %s of Component %s could not be found" % \ raise ImportError("%s: version %s of Component %s could not be found" % \
(fullname, version, name)) (fullname, version, name))
# Otherwise, find the Component with the highest version priority # Otherwise, find the Component with the highest version priority
else:
try:
component_version_dict = self._registry_dict[name]
except KeyError:
raise ImportError("%s: Component %s could not be found" % (fullname,
name))
# Version priority name list is ordered in descending order
for version in site.getVersionPriorityNameList():
component_id_uid_tuple = component_version_dict.get(version)
if component_id_uid_tuple is not None:
component_id = component_id_uid_tuple[0]
break
else: else:
try: raise ImportError("%s: no version of Component %s in Site priority" % \
component_version_dict = self._registry_dict[name] (fullname, name))
except KeyError:
raise ImportError("%s: Component %s could not be found" % (fullname,
name))
# Version priority name list is ordered in descending order
for version in site.getVersionPriorityNameList():
component_id_uid_tuple = component_version_dict.get(version)
if component_id_uid_tuple is not None:
component_id = component_id_uid_tuple[0]
break
else:
raise ImportError("%s: no version of Component %s in Site priority" % \
(fullname, name))
# Check whether this module has already been loaded before for a
# specific version, if so, just add it to the upper level
try:
module = getattr(getattr(self, version + '_version'), name)
except AttributeError:
pass
else:
setattr(self, name, module)
return module
module_fullname_alias = self._namespace + '.' + name # Check whether this module has already been loaded before for a
# specific version, if so, just add it to the upper level
try:
module = getattr(getattr(self, version + '_version'), name)
except AttributeError:
pass
else:
setattr(self, name, module)
return module
component = getattr(site.portal_components, component_id) module_fullname_alias = self._namespace + '.' + name
relative_url = component.getRelativeUrl()
module_fullname = '%s.%s_version.%s' % (self._namespace, version, name) component = getattr(site.portal_components, component_id)
module = ModuleType(module_fullname, component.getDescription()) relative_url = component.getRelativeUrl()
source_code_str = component.getTextContent(validated_only=True) module_fullname = '%s.%s_version.%s' % (self._namespace, version, name)
version_package = self._getVersionPackage(version) module = ModuleType(module_fullname, component.getDescription())
finally: source_code_str = component.getTextContent(validated_only=True)
# Internal release of import lock at the end of import machinery will version_package = self._getVersionPackage(version)
# fail if the hook is not acquired
if import_lock_held:
imp.acquire_lock()
# All the required objects have been loaded, acquire import lock to modify # All the required objects have been loaded, acquire import lock to modify
# sys.modules and execute PEP302 requisites # sys.modules and execute PEP302 requisites
if not import_lock_held: imp.acquire_lock()
imp.acquire_lock()
try: try:
# The module *must* be in sys.modules before executing the code in case # The module *must* be in sys.modules before executing the code in case
# the module code imports (directly or indirectly) itself (see PEP 302) # the module code imports (directly or indirectly) itself (see PEP 302)
...@@ -393,19 +367,41 @@ class ComponentDynamicPackage(ModuleType): ...@@ -393,19 +367,41 @@ class ComponentDynamicPackage(ModuleType):
return module return module
finally: finally:
# load_module() can be called outside of import machinery, for example imp.release_lock()
# to check first if the module can be handled by Component and then try
# to load it without going through the same code again
if not import_lock_held:
imp.release_lock()
def load_module(self, fullname): def load_module(self, fullname):
""" """
Make sure that loading module is thread-safe using aq_method_lock to make Make sure that loading module is thread-safe using aq_method_lock to make
sure that modules do not disappear because of an ongoing reset sure that modules do not disappear because of an ongoing reset
""" """
with aq_method_lock: # In Python < 3.3, the import lock is a global lock for all modules:
# http://bugs.python.org/issue9260
#
# So, release the import lock acquired by import statement on all hooks to
# load objects from ZODB. When an object is requested from ZEO, it sends a
# RPC request and lets the asyncore thread gets the reply. This reply may
# be a tuple (PICKLE, TID), sent directly to the first thread, or an
# Exception, which tries to import a ZODB module and thus creates a
# deadlock because of the global import lock
#
# Also, handle the case where find_module() may be called without import
# statement as it does not change anything in sys.modules
import_lock_held = True
try:
imp.release_lock()
except RuntimeError:
import_lock_held = False
aq_method_lock.acquire()
try:
return self.__load_module(fullname) return self.__load_module(fullname)
finally:
aq_method_lock.release()
# Internal release of import lock at the end of import machinery will
# fail if the hook is not acquired
if import_lock_held:
imp.acquire_lock()
def find_load_module(self, name): def find_load_module(self, name):
""" """
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment