Extract most compat code into ZODB._compat

src/ZODB/BaseStorage.py

@@ -25,27 +25,13 @@ import sys
 from struct import pack as _structpack, unpack as _structunpack
 
 import zope.interface
-
 from persistent.TimeStamp import TimeStamp
 
 import ZODB.interfaces
 from ZODB import POSException
 from ZODB.utils import z64, oid_repr, byte_ord, byte_chr
 from ZODB.UndoLogCompatible import UndoLogCompatible
-
-try:
-    import cPickle as pickle
-except ImportError:
-    # Py3
-    import pickle
-
-# Py3: Python 3's `hasattr()` only swallows AttributeError.
-def py2_hasattr(obj, name):
-    try:
-        getattr(obj, name)
-    except:
-        return False
-    return True
+from ZODB._compat import pickle, py2_hasattr
 
 log = logging.getLogger("ZODB.BaseStorage")
 

src/ZODB/ConflictResolution.py

@@ -14,25 +14,18 @@
 
 import logging
 import sys
-from pickle import PicklingError
 
 import six
 import zope.interface
 from ZODB.POSException import ConflictError
 from ZODB.loglevels import BLATHER
 from ZODB.serialize import _protocol, _Unpickler
+from ZODB._compat import BytesIO, pickle
 
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    # Py3
-    from io import BytesIO
-
-try:
-    from cPickle import Pickler
-except ImportError:
-    # Py3
-    from pickle import Pickler
+# Subtle: Python 2.x has pickle.PicklingError and cPickle.PicklingError,
+# and these are unrelated classes!  So we shouldn't use pickle.PicklingError,
+# since on Python 2, ZODB._compat.pickle is cPickle.
+from pickle import PicklingError
 
 
 logger = logging.getLogger('ZODB.ConflictResolution')
@@ -290,7 +283,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
         resolved = resolve(old, committed, newstate)
 
         file = BytesIO()
-        pickler = Pickler(file, _protocol)
+        pickler = pickle.Pickler(file, _protocol)
         if sys.version_info[0] < 3:
             pickler.inst_persistent_id = persistent_id
         else:

src/ZODB/DB.py

@@ -23,6 +23,7 @@ import warnings
 from ZODB.broken import find_global
 from ZODB.utils import z64
 from ZODB.Connection import Connection
+from ZODB._compat import pickle, BytesIO
 import ZODB.serialize
 
 import transaction.weakset
@@ -36,17 +37,6 @@ import transaction
 from persistent.TimeStamp import TimeStamp
 import six
 
-try:
-    import cPickle as pickle
-except ImportError:
-    # Py3
-    import pickle
-
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    # Py3
-    from io import BytesIO
 
 logger = logging.getLogger('ZODB.DB')
 

src/ZODB/ExportImport.py

@@ -12,30 +12,21 @@
 #
 ##############################################################################
 """Support for database export and import."""
-import os
 
-from tempfile import TemporaryFile
 import logging
-import six
+import os
 import sys
+from tempfile import TemporaryFile
+
+import six
 
 from ZODB.blob import Blob
 from ZODB.interfaces import IBlobStorage
 from ZODB.POSException import ExportError
 from ZODB.serialize import referencesf, _protocol
 from ZODB.utils import p64, u64, cp, mktemp
+from ZODB._compat import pickle, BytesIO
 
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    # Py3
-    from io import BytesIO
-
-try:
-    from cPickle import Unpickler, Pickler
-except ImportError:
-    # Py3
-    from pickle import Unpickler, Pickler
 
 logger = logging.getLogger('ZODB.ExportImport')
 
@@ -176,11 +167,11 @@ class ExportImport:
                 blob_filename = None
 
             pfile = BytesIO(data)
-            unpickler = Unpickler(pfile)
+            unpickler = pickle.Unpickler(pfile)
             unpickler.persistent_load = persistent_load
 
             newp = BytesIO()
-            pickler = Pickler(newp, _protocol)
+            pickler = pickle.Pickler(newp, _protocol)
             if sys.version_info[0] < 3:
                 pickler.inst_persistent_id = persistent_id
             else:

src/ZODB/FileStorage/FileStorage.py

@@ -15,9 +15,23 @@
 """
 from __future__ import print_function
 
-from persistent.TimeStamp import TimeStamp
+import binascii
+import contextlib
+import errno
+import logging
+import os
+import threading
+import time
 from struct import pack, unpack
+
+import six
+import zope.interface
+from persistent.TimeStamp import TimeStamp
 from zc.lockfile import LockFile
+
+import ZODB.blob
+import ZODB.interfaces
+import ZODB.utils
 from ZODB.FileStorage.format import CorruptedError, CorruptedDataError
 from ZODB.FileStorage.format import FileStorageFormatter, DataHeader
 from ZODB.FileStorage.format import TRANS_HDR, TRANS_HDR_LEN
@@ -27,31 +41,7 @@ from ZODB.fsIndex import fsIndex
 from ZODB import BaseStorage, ConflictResolution, POSException
 from ZODB.POSException import UndoError, POSKeyError, MultipleUndoErrors
 from ZODB.utils import p64, u64, z64, as_bytes, as_text
-
-import binascii
-import contextlib
-import errno
-import logging
-import os
-import six
-import threading
-import time
-import ZODB.blob
-import ZODB.interfaces
-import zope.interface
-import ZODB.utils
-
-try:
-    from cPickle import Pickler, loads
-except ImportError:
-    # Py3
-    from pickle import Pickler, loads
-
-try:
-    # Py3
-    from base64 import decodebytes, encodebytes
-except ImportError:
-    from base64 import decodestring as decodebytes, encodestring as encodebytes
+from ZODB._compat import pickle, decodebytes, encodebytes
 
 
 # Not all platforms have fsync
@@ -379,7 +369,7 @@ class FileStorage(
             if not self._is_read_only:
                 # Save the converted index.
                 f = open(index_name, 'wb')
-                p = Pickler(f, 1)
+                p = pickle.Pickler(f, 1)
                 info['index'] = index
                 p.dump(info)
                 f.close()
@@ -1011,7 +1001,7 @@ class FileStorage(
 
                 th = self._read_txn_header(h.tloc)
                 if th.ext:
-                    d = loads(th.ext)
+                    d = pickle.loads(th.ext)
                 else:
                     d = {}
 
@@ -1852,7 +1842,7 @@ class FileIterator(FileStorageFormatter):
                 e = {}
                 if h.elen:
                     try:
-                        e = loads(h.ext)
+                        e = pickle.loads(h.ext)
                     except:
                         pass
 
@@ -1994,7 +1984,7 @@ class UndoSearch:
         e = {}
         if el:
             try:
-                e = loads(self.file.read(el))
+                e = pickle.loads(self.file.read(el))
             except:
                 pass
         d = {'id': encodebytes(tid).rstrip(),

src/ZODB/_compat.py

+##############################################################################
+#
+# Copyright (c) 2013 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE
+#
+##############################################################################
+
+try:
+    # Python 2.x
+    import cPickle as pickle
+    IMPORT_MAPPING = {}
+    NAME_MAPPING = {}
+except ImportError:
+    # Python 3.x: can't use stdlib's pickle because
+    # http://bugs.python.org/issue6784
+    ## import zodbpickle as pickle
+    import pickle
+    from _compat_pickle import IMPORT_MAPPING, NAME_MAPPING
+
+
+# XXX: overridable Unpickler.find_global as used in serialize.py?
+# XXX: consistent spelling of inst_persistent_id/persistent_id?
+#      e.g. StorageTestBase and probably elsewhere
+
+
+try:
+    # Python 2.x
+    # XXX: why not just import BytesIO from io?
+    from cStringIO import StringIO as BytesIO
+except ImportError:
+    # Python 3.x
+    from io import BytesIO
+
+
+try:
+    # Python 3.x
+    from base64 import decodebytes, encodebytes
+except ImportError:
+    # Python 2.x
+    from base64 import decodestring as decodebytes, encodestring as encodebytes
+
+
+# Python 3.x: ``hasattr()`` swallows only AttributeError.
+def py2_hasattr(obj, name):
+    try:
+        getattr(obj, name)
+    except:
+        return False
+    return True
+
+
+try:
+    # Py2: simply reexport the builtin
+    long = long
+except NameError:
+    # Py3
+    long = int
+

src/ZODB/blob.py

@@ -26,28 +26,19 @@ import tempfile
 import weakref
 
 import zope.interface
+import persistent
 
 import ZODB.interfaces
 from ZODB.interfaces import BlobError
 from ZODB import utils, serialize
 from ZODB.POSException import POSKeyError
-import persistent
+from ZODB._compat import BytesIO
 
-try:
-    import cPickle
-except ImportError:
-    # Py3
-    import pickle as cPickle
-
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    # Py3
-    from io import BytesIO
 
 if sys.version_info[0] >= 3:
     from io import FileIO as file
 
+
 logger = logging.getLogger('ZODB.blob')
 
 BLOB_SUFFIX = ".blob"

src/ZODB/broken.py

@@ -15,18 +15,13 @@
 """
 
 import sys
-import persistent
 
+import persistent
 import zope.interface
 
 import ZODB.interfaces
+from ZODB._compat import pickle, IMPORT_MAPPING, NAME_MAPPING
 
-try:
-    # Python 3
-    import _compat_pickle
-except ImportError:
-    # Python 2
-    _compat_pickle = None
 
 broken_cache = {}
 
@@ -87,12 +82,7 @@ class Broken(object):
          >>> r[2]
          {'x': 1}
 
-         >>> try:
-         ...     import cPickle
-         ... except ImportError:
-         ...     # Py3
-         ...     import pickle as cPickle
-         >>> a2 = cPickle.loads(cPickle.dumps(a, 1))
+         >>> a2 = pickle.loads(pickle.dumps(a, 1))
          >>> a2
          <broken not.there.Atall instance>
          >>> a2.__Broken_newargs__
@@ -196,12 +186,10 @@ def find_global(modulename, globalname,
          >>> broken_cache.clear()
        """
 
-    if _compat_pickle is not None:
-        if (modulename, globalname) in _compat_pickle.NAME_MAPPING:
-            modulename, globalname = _compat_pickle.NAME_MAPPING[
-                                            (modulename, globalname)]
-        if modulename in _compat_pickle.IMPORT_MAPPING:
-            modulename = _compat_pickle.IMPORT_MAPPING[modulename]
+    if (modulename, globalname) in NAME_MAPPING:
+        modulename, globalname = NAME_MAPPING[(modulename, globalname)]
+    if modulename in IMPORT_MAPPING:
+        modulename = IMPORT_MAPPING[modulename]
 
     # short circuit common case:
     try:

src/ZODB/fsIndex.py

@@ -44,11 +44,7 @@ from BTrees._fsBTree import fsBucket
 from BTrees.OOBTree import OOBTree
 import six
 
-try:
-    import cPickle as pickle
-except ImportError:
-    # Py3
-    import pickle
+from ZODB._compat import pickle
 
 
 # convert between numbers and six-byte strings

src/ZODB/fsrecover.py

@@ -71,12 +71,6 @@ import getopt
 import time
 from struct import unpack
 
-try:
-    from cPickle import loads
-except ImportError:
-    # Py3
-    from pickle import loads
-
 try:
     import ZODB
 except ImportError:
@@ -89,6 +83,7 @@ except ImportError:
 import ZODB.FileStorage
 from ZODB.utils import u64, as_text
 from ZODB.FileStorage import TransactionRecord
+from ZODB._compat import pickle
 
 from persistent.TimeStamp import TimeStamp
 
@@ -149,9 +144,9 @@ def read_txn_header(f, pos, file_size, outp, ltid):
     user = f.read(ul)
     description = f.read(dl)
     if el:
-        try: e=loads(f.read(el))
-        except: e={}
-    else: e={}
+        try: e = pickle.loads(f.read(el))
+        except: e = {}
+    else: e = {}
 
     result = TransactionRecord(tid, status, user, description, e, pos, tend,
                                f, tpos)

src/ZODB/fstools.py

@@ -23,14 +23,9 @@ import struct
 from ZODB.FileStorage.format import TRANS_HDR, DATA_HDR, TRANS_HDR_LEN
 from ZODB.FileStorage.format import DATA_HDR_LEN
 from ZODB.utils import u64
+from ZODB._compat import pickle
 from persistent.TimeStamp import TimeStamp
 
-try:
-    import cPickle
-except ImportError:
-    # Py3
-    import pickle as cPickle
-
 
 class TxnHeader:
     """Object representing a transaction record header.
@@ -70,7 +65,7 @@ class TxnHeader:
             self.descr = self._file.read(self.descr_len)
         if self.ext_len:
             self._ext = self._file.read(self.ext_len)
-            self.ext = cPickle.loads(self._ext)
+            self.ext = pickle.loads(self._ext)
 
     def get_data_offset(self):
         return (self._pos + TRANS_HDR_LEN + self.user_len + self.descr_len

src/ZODB/scripts/analyze.py

@@ -2,15 +2,16 @@
 
 # Based on a transaction analyzer by Matt Kromer.
 from __future__ import print_function
-import pickle
+
 import sys
+
 from ZODB.FileStorage import FileStorage
+from ZODB._compat import BytesIO
+
+# We must not use cPickle on Python 2: cPickle.Unpickler cannot be
+# subclassed.
+import pickle
 
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    # Py3
-    from io import BytesIO
 
 class FakeError(Exception):
     def __init__(self, module, name):

src/ZODB/serialize.py

@@ -140,18 +140,7 @@ from persistent import Persistent
 from persistent.wref import WeakRefMarker, WeakRef
 from ZODB import broken
 from ZODB.POSException import InvalidObjectReference
-
-try:
-    import cPickle as pickle
-except ImportError:
-    # Py3
-    import pickle
-
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    # Py3
-    from io import BytesIO
+from ZODB._compat import pickle, BytesIO
 
 if sys.version_info[0] < 3:
     _Unpickler = pickle.Unpickler

src/ZODB/tests/PackableStorage.py

@@ -14,6 +14,10 @@
 """Run some tests relevant for storages that support pack()."""
 from __future__ import print_function
 
+import doctest
+import sys
+import time
+
 from persistent import Persistent
 from persistent.mapping import PersistentMapping
 from ZODB import DB
@@ -22,25 +26,12 @@ from ZODB.serialize import referencesf, _Unpickler, _protocol
 from ZODB.tests.MinPO import MinPO
 from ZODB.tests.MTStorage import TestThread
 from ZODB.tests.StorageTestBase import snooze
-import doctest
-import sys
-import time
+from ZODB._compat import pickle, BytesIO
 import transaction
 import ZODB.interfaces
 import ZODB.tests.util
 import zope.testing.setupstack
 
-try:
-    import cPickle as pickle
-except ImportError:
-    # Py3
-    import pickle
-
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    # Py3
-    from io import BytesIO
 
 ZERO = b'\0'*8
 

src/ZODB/tests/StorageTestBase.py

@@ -25,20 +25,9 @@ import transaction
 
 from ZODB.utils import u64
 from ZODB.tests.MinPO import MinPO
+from ZODB._compat import pickle, BytesIO
 import ZODB.tests.util
 
-try:
-    from cPickle import Pickler, Unpickler
-except ImportError:
-    # Py3
-    from pickle import Pickler, Unpickler
-
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    # Py3
-    from io import BytesIO
-
 
 ZERO = b'\0'*8
 
@@ -61,7 +50,7 @@ def _persistent_id(obj):
 def zodb_pickle(obj):
     """Create a pickle in the format expected by ZODB."""
     f = BytesIO()
-    p = Pickler(f, 1)
+    p = pickle.Pickler(f, 1)
     if sys.version_info[0] < 3:
         p.inst_persistent_id = _persistent_id
     else:
@@ -87,7 +76,7 @@ def persistent_load(pid):
 def zodb_unpickle(data):
     """Unpickle an object stored using the format expected by ZODB."""
     f = BytesIO(data)
-    u = Unpickler(f)
+    u = pickle.Unpickler(f)
     u.persistent_load = persistent_load
     klass_info = u.load()
     if isinstance(klass_info, tuple):

src/ZODB/tests/testFileStorage.py

@@ -32,13 +32,7 @@ from ZODB.tests import HistoryStorage, IteratorStorage, Corruption
 from ZODB.tests import RevisionStorage, PersistentStorage, MTStorage
 from ZODB.tests import ReadOnlyStorage, RecoveryStorage
 from ZODB.tests.StorageTestBase import MinPO, zodb_pickle
-
-
-try:
-    import cPickle
-except ImportError:
-    # Py3
-    import pickle as cPickle
+from ZODB._compat import pickle
 
 
 class FileStorageTests(
@@ -97,7 +91,7 @@ class FileStorageTests(
         newindex = dict(index)
         data['index'] = newindex
 
-        cPickle.dump(data, open('FileStorageTests.fs.index', 'wb'), 1)
+        pickle.dump(data, open('FileStorageTests.fs.index', 'wb'), 1)
         return index
 
     def check_conversion_to_fsIndex(self, read_only=False):

src/ZODB/tests/testPersistentMapping.py

@@ -21,18 +21,14 @@ old code, developers will have a hard time testing the new code.
 """
 
 import unittest
+import sys
 
 import transaction
 from transaction import Transaction
+
 import ZODB
 from ZODB.MappingStorage import MappingStorage
-import sys
-
-try:
-    import cPickle
-except ImportError:
-    # Py3
-    import pickle as cPickle
+from ZODB._compat import pickle
 
 try:
     import cStringIO
@@ -89,8 +85,9 @@ class PMTests(unittest.TestCase):
         # MappingStorage stores serialno + pickle in its _index.
         root_pickle = s._index['\000' * 8][8:]
 
+        # XXX not BytesIO really?
         f = cStringIO.StringIO(root_pickle)
-        u = cPickle.Unpickler(f)
+        u = pickle.Unpickler(f)
         klass_info = u.load()
         klass = find_global(*klass_info[0])
         inst = klass.__new__(klass)

src/ZODB/tests/testSerialize.py

@@ -17,18 +17,7 @@ import unittest
 
 import ZODB.tests.util
 from ZODB import serialize
-
-try:
-    import cPickle as pickle
-except ImportError:
-    # Py3
-    import pickle
-
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    # Py3
-    from io import BytesIO
+from ZODB._compat import pickle, BytesIO
 
 
 class ClassWithNewargs(int):

src/ZODB/tests/testUtils.py

@@ -20,12 +20,8 @@ from persistent import Persistent
 
 from zope.testing import renormalizing
 from ZODB.utils import U64, p64, u64
+from ZODB._compat import pickle, long
 
-try:
-    long
-except NameError:
-    # Py3
-    long = int
 
 NUM = 100
 
@@ -77,11 +73,6 @@ class TestUtils(unittest.TestCase):
         from ZODB.serialize import ObjectWriter
         from ZODB.POSException import ConflictError
         from ZODB.tests.MinPO import MinPO
-        try:
-            import cPickle as pickle
-        except ImportError:
-            # Py3
-            import pickle
 
         obj = MinPO()
         data = ObjectWriter().serialize(obj)

src/ZODB/tests/testfsIndex.py

@@ -220,12 +220,8 @@ Note that we pass a file position, which gets saved with the index data.
 
 If we save the data in the old format, we can still read it:
 
-    >>> try:
-    ...     import cPickle
-    ... except ImportError:
-    ...     # Py3
-    ...     import pickle as cPickle
-    >>> cPickle.dump(dict(pos=42, index=index), open('old', 'wb'), 1)
+    >>> from ZODB._compat import pickle
+    >>> pickle.dump(dict(pos=42, index=index), open('old', 'wb'), 1)
     >>> info = fsIndex.load('old')
     >>> info['pos']
     42

src/ZODB/utils.py

@@ -11,29 +11,19 @@
 # FOR A PARTICULAR PURPOSE
 #
 ##############################################################################
+import os
+import struct
 import sys
 import time
-import struct
-from struct import pack, unpack
-from binascii import hexlify, unhexlify
 import warnings
+from binascii import hexlify, unhexlify
+from struct import pack, unpack
 from tempfile import mkstemp
-import os
-
-try:
-    import cPickle as pickle
-except ImportError:
-    # Py3
-    import pickle
 
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    # Py3
-    from io import BytesIO
+from persistent.TimeStamp import TimeStamp
 
+from ZODB._compat import pickle, BytesIO
 
-from persistent.TimeStamp import TimeStamp
 
 __all__ = ['z64',
            'p64',