Commit 8c9ed0db authored by Jason Madden's avatar Jason Madden

Fix a UnicodeDecodeError that could happen with blobs when the OID contained...

Fix a UnicodeDecodeError that could happen with blobs when the OID contained non-ascii bytes if ascii was the default encoding; test this.
parent 6dc3a3e0
...@@ -2,6 +2,12 @@ ...@@ -2,6 +2,12 @@
Change History Change History
================ ================
Unreleased
==========
- Fixed: A ``UnicodeDecodeError`` could happen for non-ASCII OIDs
when using bushy blob layout.
4.0.0b2 (2013-05-14) 4.0.0b2 (2013-05-14)
==================== ====================
......
...@@ -35,6 +35,7 @@ from ZODB._compat import BytesIO ...@@ -35,6 +35,7 @@ from ZODB._compat import BytesIO
from ZODB._compat import Unpickler from ZODB._compat import Unpickler
from ZODB._compat import decodebytes from ZODB._compat import decodebytes
from ZODB._compat import ascii_bytes from ZODB._compat import ascii_bytes
from ZODB._compat import INT_TYPES
if sys.version_info[0] >= 3: if sys.version_info[0] >= 3:
...@@ -552,9 +553,14 @@ class BushyLayout(object): ...@@ -552,9 +553,14 @@ class BushyLayout(object):
directories = [] directories = []
# Create the bushy directory structure with the least significant byte # Create the bushy directory structure with the least significant byte
# first # first
for byte in oid.decode(): for byte in ascii_bytes(oid):
directories.append( if isinstance(byte,INT_TYPES): # Py3k iterates byte strings as ints
'0x%s' % binascii.hexlify(byte.encode()).decode()) hex_segment_bytes = b'0x' + binascii.hexlify(bytes([byte]))
hex_segment_string = hex_segment_bytes.decode('ascii')
else:
hex_segment_string = '0x%s' % binascii.hexlify(byte)
directories.append(hex_segment_string)
return os.path.sep.join(directories) return os.path.sep.join(directories)
def path_to_oid(self, path): def path_to_oid(self, path):
......
...@@ -12,6 +12,7 @@ ...@@ -12,6 +12,7 @@
# #
############################################################################## ##############################################################################
from ZODB.blob import Blob from ZODB.blob import Blob
from ZODB.blob import BushyLayout
from ZODB.DB import DB from ZODB.DB import DB
from ZODB.FileStorage import FileStorage from ZODB.FileStorage import FileStorage
from ZODB.tests.testConfig import ConfigTestBase from ZODB.tests.testConfig import ConfigTestBase
...@@ -138,6 +139,28 @@ class BlobCloneTests(ZODB.tests.util.TestCase): ...@@ -138,6 +139,28 @@ class BlobCloneTests(ZODB.tests.util.TestCase):
# tearDown # tearDown
database.close() database.close()
class BushyLayoutTests(ZODB.tests.util.TestCase):
def testBushyLayoutOIDToPathUnicode(self):
"OID-to-path should produce valid results given non-ASCII byte strings"
non_ascii_oid = b'>\xf1<0\xe9Q\x99\xf0'
# The argument should already be bytes;
# os.path.sep is native string type under both 2 and 3
# binascii.hexlify takes bytes and produces bytes under both py2 and py3
# the result should be the native string type
oid_as_path = BushyLayout().oid_to_path(non_ascii_oid)
self.assertEqual(
oid_as_path,
'0x3e/0xf1/0x3c/0x30/0xe9/0x51/0x99/0xf0')
# the reverse holds true as well
path_as_oid = BushyLayout().path_to_oid(oid_as_path)
self.assertEqual(
path_as_oid,
non_ascii_oid )
class BlobTestBase(ZODB.tests.StorageTestBase.StorageTestBase): class BlobTestBase(ZODB.tests.StorageTestBase.StorageTestBase):
def setUp(self): def setUp(self):
...@@ -779,6 +802,7 @@ def test_suite(): ...@@ -779,6 +802,7 @@ def test_suite():
suite = unittest.TestSuite() suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(ZODBBlobConfigTest)) suite.addTest(unittest.makeSuite(ZODBBlobConfigTest))
suite.addTest(unittest.makeSuite(BlobCloneTests)) suite.addTest(unittest.makeSuite(BlobCloneTests))
suite.addTest(unittest.makeSuite(BushyLayoutTests))
suite.addTest(doctest.DocFileSuite( suite.addTest(doctest.DocFileSuite(
"blob_basic.txt", "blob_consume.txt", "blob_tempdir.txt", "blob_basic.txt", "blob_consume.txt", "blob_tempdir.txt",
"blobstorage_packing.txt", "blobstorage_packing.txt",
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment