- ZODB now uses pickle protocol 3 for both Python 2 and Python 3. (#194)

(Previously, protocol 2 was used for Python 2.)

  The zodbpickle package provides a `zodbpickle.binary` string type
  that should be used in Python 2 to cause binary strings to be saved
  in a pickle binary format, so they can be loaded correctly in
  Python 3.  Pickle protocol 3 is needed for this to work correctly.

- Object identifiers in persistent references are saved as
  `zodbpickle.binary` strings in Python 2, so that they are loaded
  correctly in Python 3.

CHANGES.rst

@@ -5,6 +5,19 @@
 5.4.0 (unreleased)
 ==================
 
+- ZODB now uses pickle protocol 3 for both Python 2 and Python 3.
+
+  (Previously, protocol 2 was used for Python 2.)
+
+  The zodbpickle package provides a `zodbpickle.binary` string type
+  that should be used in Python 2 to cause binary strings to be saved
+  in a pickle binary format, so they can be loaded correctly in
+  Python 3.  Pickle protocol 3 is needed for this to work correctly.
+
+- Object identifiers in persistent references are saved as
+  `zodbpickle.binary` strings in Python 2, so that they are loaded
+  correctly in Python 3.
+
 - If an object is missing from the index while packing a ``FileStorage``,
   report its full ``oid``.
 

src/ZODB/_compat.py

@@ -16,6 +16,8 @@ from six import PY3
 
 IS_JYTHON = sys.platform.startswith('java')
 
+_protocol = 3
+from zodbpickle import binary
 
 if not PY3:
     # Python 2.x
@@ -34,7 +36,6 @@ if not PY3:
     HIGHEST_PROTOCOL = cPickle.HIGHEST_PROTOCOL
     IMPORT_MAPPING = {}
     NAME_MAPPING = {}
-    _protocol = 2
     FILESTORAGE_MAGIC = b"FS21"
 else:
     # Python 3.x: can't use stdlib's pickle because
@@ -69,7 +70,6 @@ else:
 
     def loads(s):
         return zodbpickle.pickle.loads(s, encoding='ASCII', errors='bytes')
-    _protocol = 3
     FILESTORAGE_MAGIC = b"FS30"
 
 

src/ZODB/serialize.py

@@ -139,7 +139,8 @@ from persistent import Persistent
 from persistent.wref import WeakRefMarker, WeakRef
 from ZODB import broken
 from ZODB.POSException import InvalidObjectReference
-from ZODB._compat import PersistentPickler, PersistentUnpickler, BytesIO, _protocol
+from ZODB._compat import PersistentPickler, PersistentUnpickler, BytesIO
+from ZODB._compat import _protocol, binary
 
 
 _oidtypes = bytes, type(None)
@@ -186,7 +187,7 @@ class ObjectWriter(object):
         >>> class DummyJar(object):
         ...     xrefs = True
         ...     def new_oid(self):
-        ...         return 42
+        ...         return b'42'
         ...     def db(self):
         ...         return self
         ...     databases = {}
@@ -204,24 +205,31 @@ class ObjectWriter(object):
         >>> bob = P('bob')
         >>> oid, cls = writer.persistent_id(bob)
         >>> oid
-        42
+        '42'
         >>> cls is P
         True
 
+        To work with Python 3, the oid in the persistent id is of the
+        zodbpickle binary type:
+
+        >>> oid.__class__ is binary
+        True
+
+
         If a persistent object does not already have an oid and jar,
         these will be assigned by persistent_id():
 
         >>> bob._p_oid
-        42
+        '42'
         >>> bob._p_jar is jar
         True
 
         If the object already has a persistent id, the id is not changed:
 
-        >>> bob._p_oid = 24
+        >>> bob._p_oid = b'24'
         >>> oid, cls = writer.persistent_id(bob)
         >>> oid
-        24
+        '24'
         >>> cls is P
         True
 
@@ -247,9 +255,9 @@ class ObjectWriter(object):
 
         >>> sam = PNewArgs('sam')
         >>> writer.persistent_id(sam)
-        42
+        '42'
         >>> sam._p_oid
-        42
+        '42'
         >>> sam._p_jar is jar
         True
 
@@ -312,6 +320,8 @@ class ObjectWriter(object):
                     obj.oid = oid
                     obj.dm = target._p_jar
                     obj.database_name = obj.dm.db().database_name
+
+                oid = binary(oid)
                 if obj.dm is self._jar:
                     return ['w', (oid, )]
                 else:
@@ -366,6 +376,7 @@ class ObjectWriter(object):
                     self._jar, obj,
                     )
 
+        oid = binary(oid)
         klass = type(obj)
         if hasattr(klass, '__getnewargs__'):
             # We don't want to save newargs in object refs.

src/ZODB/tests/testSerialize.py

@@ -137,6 +137,9 @@ class SerializerTestCase(unittest.TestCase):
         top.ref = WeakRef(o)
 
         pickle = serialize.ObjectWriter().serialize(top)
+        # Make sure the persistent id is pickled using the 'C',
+        # SHORT_BINBYTES opcode:
+        self.assertTrue(b'C\x04abcd' in pickle)
 
         refs = []
         u = PersistentUnpickler(None, refs.append, BytesIO(pickle))
@@ -145,6 +148,18 @@ class SerializerTestCase(unittest.TestCase):
 
         self.assertEqual(refs, [['w', (b'abcd',)]])
 
+    def test_protocol_3_binary_handling(self):
+        from ZODB.serialize import _protocol
+        self.assertEqual(3, _protocol) # Yeah, whitebox
+        o = PersistentObject()
+        o._p_oid = b'o'
+        o.o = PersistentObject()
+        o.o._p_oid = b'o.o'
+        pickle = serialize.ObjectWriter().serialize(o)
+
+        # Make sure the persistent id is pickled using the 'C',
+        # SHORT_BINBYTES opcode:
+        self.assertTrue(b'C\x03o.o' in pickle)
 
 class SerializerFunctestCase(unittest.TestCase):