Commit e2543bbf authored by Tres Seaver's avatar Tres Seaver

Merge pull request #9 from NextThought/master

UnicodeDecodeError with bushy blob layout and non-ASCII OID
parents 6dc3a3e0 8c9ed0db
...@@ -2,6 +2,12 @@ ...@@ -2,6 +2,12 @@
Change History Change History
================ ================
Unreleased
==========
- Fixed: A ``UnicodeDecodeError`` could happen for non-ASCII OIDs
when using bushy blob layout.
4.0.0b2 (2013-05-14) 4.0.0b2 (2013-05-14)
==================== ====================
......
...@@ -35,6 +35,7 @@ from ZODB._compat import BytesIO ...@@ -35,6 +35,7 @@ from ZODB._compat import BytesIO
from ZODB._compat import Unpickler from ZODB._compat import Unpickler
from ZODB._compat import decodebytes from ZODB._compat import decodebytes
from ZODB._compat import ascii_bytes from ZODB._compat import ascii_bytes
from ZODB._compat import INT_TYPES
if sys.version_info[0] >= 3: if sys.version_info[0] >= 3:
...@@ -552,9 +553,14 @@ class BushyLayout(object): ...@@ -552,9 +553,14 @@ class BushyLayout(object):
directories = [] directories = []
# Create the bushy directory structure with the least significant byte # Create the bushy directory structure with the least significant byte
# first # first
for byte in oid.decode(): for byte in ascii_bytes(oid):
directories.append( if isinstance(byte,INT_TYPES): # Py3k iterates byte strings as ints
'0x%s' % binascii.hexlify(byte.encode()).decode()) hex_segment_bytes = b'0x' + binascii.hexlify(bytes([byte]))
hex_segment_string = hex_segment_bytes.decode('ascii')
else:
hex_segment_string = '0x%s' % binascii.hexlify(byte)
directories.append(hex_segment_string)
return os.path.sep.join(directories) return os.path.sep.join(directories)
def path_to_oid(self, path): def path_to_oid(self, path):
......
...@@ -12,6 +12,7 @@ ...@@ -12,6 +12,7 @@
# #
############################################################################## ##############################################################################
from ZODB.blob import Blob from ZODB.blob import Blob
from ZODB.blob import BushyLayout
from ZODB.DB import DB from ZODB.DB import DB
from ZODB.FileStorage import FileStorage from ZODB.FileStorage import FileStorage
from ZODB.tests.testConfig import ConfigTestBase from ZODB.tests.testConfig import ConfigTestBase
...@@ -138,6 +139,28 @@ class BlobCloneTests(ZODB.tests.util.TestCase): ...@@ -138,6 +139,28 @@ class BlobCloneTests(ZODB.tests.util.TestCase):
# tearDown # tearDown
database.close() database.close()
class BushyLayoutTests(ZODB.tests.util.TestCase):
def testBushyLayoutOIDToPathUnicode(self):
"OID-to-path should produce valid results given non-ASCII byte strings"
non_ascii_oid = b'>\xf1<0\xe9Q\x99\xf0'
# The argument should already be bytes;
# os.path.sep is native string type under both 2 and 3
# binascii.hexlify takes bytes and produces bytes under both py2 and py3
# the result should be the native string type
oid_as_path = BushyLayout().oid_to_path(non_ascii_oid)
self.assertEqual(
oid_as_path,
'0x3e/0xf1/0x3c/0x30/0xe9/0x51/0x99/0xf0')
# the reverse holds true as well
path_as_oid = BushyLayout().path_to_oid(oid_as_path)
self.assertEqual(
path_as_oid,
non_ascii_oid )
class BlobTestBase(ZODB.tests.StorageTestBase.StorageTestBase): class BlobTestBase(ZODB.tests.StorageTestBase.StorageTestBase):
def setUp(self): def setUp(self):
...@@ -779,6 +802,7 @@ def test_suite(): ...@@ -779,6 +802,7 @@ def test_suite():
suite = unittest.TestSuite() suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(ZODBBlobConfigTest)) suite.addTest(unittest.makeSuite(ZODBBlobConfigTest))
suite.addTest(unittest.makeSuite(BlobCloneTests)) suite.addTest(unittest.makeSuite(BlobCloneTests))
suite.addTest(unittest.makeSuite(BushyLayoutTests))
suite.addTest(doctest.DocFileSuite( suite.addTest(doctest.DocFileSuite(
"blob_basic.txt", "blob_consume.txt", "blob_tempdir.txt", "blob_basic.txt", "blob_consume.txt", "blob_tempdir.txt",
"blobstorage_packing.txt", "blobstorage_packing.txt",
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment