Back out my changes that only belong onto the branch right now.

src/CHANGES.txt

@@ -8,12 +8,6 @@ Change History
 New Features
 ------------
 
-- Changed layout strategy for the blob directory to a bushy approach (8 levels
-  deep, at most ~256 entries per directory level, one directory for each
-  blob). Old directories are automatically detected and will be handled with
-  the old strategy. A migration script (`migrateblobs.py`) is provided to
-  convert the different layouts.
-
 - Versions are no-longer supported.
 
 - ZEO cache files can be larger than 4G. Note that older ZEO cache

src/ZEO/ClientStorage.py

@@ -855,7 +855,9 @@ class ClientStorage(object):
 
     def _storeBlob_shared(self, oid, serial, data, filename, txn):
         # First, move the blob into the blob directory
-        self.fshelper.getPathForOID(oid, create=True)
+        dir = self.fshelper.getPathForOID(oid)
+        if not os.path.exists(dir):
+            os.mkdir(dir)
         fd, target = self.fshelper.blob_mkstemp(oid, serial)
         os.close(fd)
 
@@ -922,7 +924,14 @@ class ClientStorage(object):
             raise POSException.POSKeyError("No blob file", oid, serial)
 
         # First, we'll create the directory for this oid, if it doesn't exist. 
-        targetpath = self.fshelper.getPathForOID(oid, create=True)
+        targetpath = self.fshelper.getPathForOID(oid)
+        if not os.path.exists(targetpath):
+            try:
+                os.makedirs(targetpath, 0700)
+            except OSError:
+                # We might have lost a race.  If so, the directory
+                # must exist now
+                assert os.path.exists(targetpath)
 
         # OK, it's not here and we (or someone) needs to get it.  We
         # want to avoid getting it multiple times.  We want to avoid
@@ -1109,15 +1118,19 @@ class ClientStorage(object):
                     assert s == tid, (s, tid)
                     self._cache.store(oid, s, None, data)
 
+        
         if self.fshelper is not None:
             blobs = self._tbuf.blobs
             while blobs:
                 oid, blobfilename = blobs.pop()
-                targetpath = self.fshelper.getPathForOID(oid, create=True)
+                targetpath = self.fshelper.getPathForOID(oid)
+                if not os.path.exists(targetpath):
+                    os.makedirs(targetpath, 0700)
                 rename_or_copy_blob(blobfilename,
                           self.fshelper.getBlobFilename(oid, tid),
                           )
 
+                    
         self._tbuf.clear()
 
     def undo(self, trans_id, txn):

src/ZEO/tests/testZEO.py

@@ -515,7 +515,8 @@ class CommonBlobTests:
             self._storage.tpc_abort(t)
             raise
         self.assert_(not os.path.exists(tfname))
-        filename = self._storage.fshelper.getBlobFilename(oid, revid)
+        filename = os.path.join(self.blobdir, oid_repr(oid),
+                                tid_repr(revid) + BLOB_SUFFIX)
         self.assert_(os.path.exists(filename))
         self.assertEqual(somedata, open(filename).read())
 
@@ -630,14 +631,16 @@ class BlobAdaptedFileStorageTests(GenericTests, CommonBlobTests):
                 d2 = somedata.read(8096)
                 self.assertEqual(d1, d2)
                 
-        # The file should be in the cache ...
-        filename = self._storage.fshelper.getBlobFilename(oid, revid)
+        
+        # The file should have been copied to the server:
+        filename = os.path.join(self.blobdir, oid_repr(oid),
+                                tid_repr(revid) + BLOB_SUFFIX)
         check_data(filename)
 
-        # ... and on the server
-        server_filename = filename.replace(self.blob_cache_dir, self.blobdir)
-        self.assert_(server_filename.startswith(self.blobdir))
-        check_data(server_filename)
+        # It should also be in the cache:
+        filename = os.path.join(self.blob_cache_dir, oid_repr(oid),
+                                tid_repr(revid) + BLOB_SUFFIX)
+        check_data(filename)
 
         # If we remove it from the cache and call loadBlob, it should
         # come back. We can do this in many threads.  We'll instrument

src/ZODB/scripts/migrateblobs.py

-##############################################################################
-#
-# Copyright (c) 2008 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-"""A script to migrate a blob directory into a different layout.
-"""
-
-import logging
-import optparse
-import os
-
-from ZODB.blob import FilesystemHelper, rename_or_copy_blob
-from ZODB.utils import cp, oid_repr
-
-
-def link_or_copy(f1, f2):
-    try:
-        os.link(f1, f2)
-    except OSError:
-        shutil.copy(f1, f2)
-
-
-def migrate(source, dest, layout):
-    source_fsh = FilesystemHelper(source)
-    source_fsh.create()
-    dest_fsh = FilesystemHelper(dest, layout)
-    dest_fsh.create()
-    print "Migrating blob data from `%s` (%s) to `%s` (%s)" % (
-        source, source_fsh.layout_name, dest, dest_fsh.layout_name)
-    for oid, path in source_fsh.listOIDs():
-        dest_path = dest_fsh.getPathForOID(oid, create=True)
-        files = os.listdir(path)
-        for file in files:
-            source_file = os.path.join(path, file)
-            dest_file = os.path.join(dest_path, file)
-            link_or_copy(source_file, dest_file)
-        print "\tOID: %s - %s files " % (oid_repr(oid), len(files))
-
-
-def main(source=None, dest=None, layout="bushy"):
-    usage = "usage: %prog [options] <source> <dest> <layout>"
-    description = ("Create the new directory <dest> and migrate all blob "
-                   "data <source> to <dest> while using the new <layout> for "
-                   "<dest>")
-
-    parser = optparse.OptionParser(usage=usage, description=description)
-    parser.add_option("-l", "--layout",
-                      default=layout, type='choice',
-                      choices=['bushy', 'lawn'],
-                      help="Define the layout to use for the new directory "
-                      "(bushy or lawn). Default: %default")
-    options, args = parser.parse_args()
-
-    if not len(args) == 2:
-        parser.error("source and destination must be given")
-
-    logging.getLogger().addHandler(logging.StreamHandler())
-    logging.getLogger().setLevel(0)
-
-    source, dest = args
-    migrate(source, dest, options.layout)
-
-
-if __name__ == '__main__':
-    main()

src/ZODB/tests/blob_tempdir.txt

@@ -32,7 +32,7 @@ First, we need a datatabase with blob support::
   >>> from ZODB.DB import DB
   >>> from tempfile import mkdtemp
   >>> import os.path
-  >>> base_storage = MappingStorage('test')
+  >>> base_storage = MappingStorage("test")
   >>> blob_dir = mkdtemp()
   >>> blob_storage = BlobStorage(blob_dir, base_storage)
   >>> database = DB(blob_storage)

src/ZODB/tests/blob_transaction.txt

@@ -322,9 +322,9 @@ clean up dirty files:
     >>> base_storage = DummyBaseStorage()
     >>> blob_dir2 = mkdtemp()
     >>> blob_storage2 = BlobStorage(blob_dir2, base_storage)
-    >>> committed_blob_dir = blob_storage2.fshelper.getPathForOID(0)
-    >>> os.makedirs(committed_blob_dir)
-    >>> committed_blob_file = blob_storage2.fshelper.getBlobFilename(0, 0)
+    >>> committed_blob_dir = os.path.join(blob_dir2, '0')
+    >>> committed_blob_file = os.path.join(committed_blob_dir, '0.blob')
+    >>> os.mkdir(committed_blob_dir)
     >>> open(os.path.join(committed_blob_file), 'w').write('foo')
     >>> os.path.exists(committed_blob_file)
     True

src/ZODB/tests/testblob.py

@@ -105,6 +105,7 @@ class BlobUndoTests(unittest.TestCase):
         self.here = os.getcwd()
         os.chdir(self.test_dir)
         self.storagefile = 'Data.fs'
+        os.mkdir('blobs')
         self.blob_dir = 'blobs'
 
     def tearDown(self):
@@ -482,7 +483,7 @@ def loadblob_tmpstore():
     We can access the blob correctly:
 
     >>> tmpstore.loadBlob(blob_oid, tid) # doctest: +ELLIPSIS
-    '.../0x01/0x00/0x00/0x00/0x00/0x00/0x00/0x00/0x...blob'
+    '.../0x01/0x...blob'
 
     Clean up:
 
@@ -506,12 +507,6 @@ def test_suite():
         setUp=ZODB.tests.util.setUp,
         tearDown=ZODB.tests.util.tearDown,
         ))
-    suite.addTest(doctest.DocFileSuite(
-        "blob_layout.txt",
-        optionflags=doctest.ELLIPSIS|doctest.NORMALIZE_WHITESPACE|doctest.REPORT_NDIFF,
-        setUp=ZODB.tests.util.setUp,
-        tearDown=ZODB.tests.util.tearDown,
-        ))
     suite.addTest(doctest.DocTestSuite(
         setUp=ZODB.tests.util.setUp,
         tearDown=ZODB.tests.util.tearDown,