branches/ctheune-blobsupport/src/ZEO/ClientStorage.py

@@ -901,6 +901,7 @@ class ClientStorage(object):
                 self._server.storeBlobEnd(oid, serial, data, version, id(txn))
                 break
             self._server.storeBlob(oid, serial, chunk, version, id(txn))
+        os.unlink(blobfilename)
         return serials
 
     def _getDirtyFilename(self, oid, serial):
@@ -917,6 +918,7 @@ class ClientStorage(object):
                                          utils.tid_repr(tid), 
                                          BLOB_SUFFIX,)
                             )
+
     def loadBlob(self, oid, serial, version):
         blob_filename = self._getCleanFilename(oid, serial)
         if os.path.exists(blob_filename):    # XXX see race condition below

branches/ctheune-blobsupport/src/ZODB/Blobs/BlobStorage.py

@@ -109,7 +109,10 @@ class BlobStorage(ProxyBase):
     def loadBlob(self, oid, serial, version):
         """Return the filename where the blob file can be found.
         """
-        return self._getCleanFilename(oid, serial)
+        filename = self._getCleanFilename(oid, serial)
+        if not os.path.exists(filename):
+            raise POSKeyError, "Not an existing blob."
+        return filename
 
     def _getNewestBlobSerial(self, oid):
         blob_path = self._getBlobPath(oid)

branches/ctheune-blobsupport/src/ZODB/Blobs/TODO.txt

 
 - Support database import/export
 
-- Support ZEO
+- Support selection of text/binary mode for opening blobs
 
-- Support selection of text/binary mode for blobs
+- Generic wrapper configuration for BlobStorage
 
-    -   implement loadBlob and storeBlob
+Tests
+-----
 
-    -   loadBlob needs to handle the BLOB_CACHE_DIRECTORY
-
-    -   storeBlob needs to hand the actual file data off to the server
+- Importing backward compatible ZEXP files (no \0BLOBSTART) used

branches/ctheune-blobsupport/src/ZODB/Blobs/interfaces.py

@@ -25,6 +25,8 @@ class IBlobStorage(Interface):
         serial.
 
         Returns a filename or None if no Blob data is connected with this OID. 
+
+        Raises POSKeyError if the blobfile cannot be found.
         """
 
     def getBlobDirectory():

branches/ctheune-blobsupport/src/ZODB/ExportImport.py

@@ -13,13 +13,16 @@
 ##############################################################################
 """Support for database export and import."""
 
+import os
+
 from cStringIO import StringIO
 from cPickle import Pickler, Unpickler
 from tempfile import TemporaryFile
 import logging
 
-from ZODB.POSException import ExportError
-from ZODB.utils import p64, u64
+from ZODB.POSException import ExportError, POSKeyError
+from ZODB.utils import p64, u64, cp, mktemp
+from ZODB.Blobs.interfaces import IBlobStorage
 from ZODB.serialize import referencesf
 
 logger = logging.getLogger('ZODB.ExportImport')
@@ -49,6 +52,21 @@ class ExportImport:
             else:
                 referencesf(p, oids)
                 f.writelines([oid, p64(len(p)), p])
+            # Blob support
+            if not IBlobStorage.providedBy(self._storage):
+                continue
+            try:
+                blobfilename = self._storage.loadBlob(oid, 
+                                                      serial, self._version)
+            except POSKeyError: # Looks like this is not a blob
+                continue
+
+            f.write(blob_begin_marker)
+            f.write(p64(os.stat(blobfilename).st_size))
+            blobdata = open(blobfilename, "rb")
+            cp(blobdata, f)
+            blobdata.close()
+            
         f.write(export_end_marker)
         return f
 
@@ -109,17 +127,20 @@ class ExportImport:
         version = self._version
 
         while 1:
-            h = f.read(16)
-            if h == export_end_marker:
+            header = f.read(16)
+            if header == export_end_marker:
                 break
-            if len(h) != 16:
+            if len(header) != 16:
                 raise ExportError("Truncated export file")
-            l = u64(h[8:16])
-            p = f.read(l)
-            if len(p) != l:
+
+            # Extract header information
+            ooid = header[:8]
+            length = u64(header[8:16])
+            data = f.read(length)
+
+            if len(data) != length:
                 raise ExportError("Truncated export file")
 
-            ooid = h[:8]
             if oids:
                 oid = oids[ooid]
                 if isinstance(oid, tuple):
@@ -128,7 +149,21 @@ class ExportImport:
                 oids[ooid] = oid = self._storage.new_oid()
                 return_oid_list.append(oid)
 
-            pfile = StringIO(p)
+            # Blob support
+            blob_begin = f.read(len(blob_begin_marker))
+            if blob_begin == blob_begin_marker:
+                # Copy the blob data to a temporary file
+                # and remember the name
+                blob_len = u64(f.read(8))
+                blob_filename = mktemp()
+                blob_file = open(blob_filename, "wb")
+                cp(f, blob_file, blob_len)
+                blob_file.close()
+            else:
+                f.seek(-len(blob_begin_marker),1)
+                blob_filename = None
+
+            pfile = StringIO(data)
             unpickler = Unpickler(pfile)
             unpickler.persistent_load = persistent_load
 
@@ -138,12 +173,17 @@ class ExportImport:
 
             pickler.dump(unpickler.load())
             pickler.dump(unpickler.load())
-            p = newp.getvalue()
+            data = newp.getvalue()
 
-            self._storage.store(oid, None, p, version, transaction)
+            if blob_filename is not None:
+                self._storage.storeBlob(oid, None, data, blob_filename, 
+                                        version, transaction)
+            else:
+                self._storage.store(oid, None, data, version, transaction)
 
 
 export_end_marker = '\377'*16
+blob_begin_marker = '\000BLOBSTART'
 
 class Ghost(object):
     __slots__ = ("oid",)

branches/ctheune-blobsupport/src/ZODB/TmpStore.py

@@ -12,6 +12,8 @@
 #
 ##############################################################################
 
+import os
+
 from zope.interface import implements
 
 from ZODB.Blobs.interfaces import IBlobStorage
@@ -150,6 +152,6 @@ class TmpStore:
     def generateBlobFile(self, oid):
         if not self.blob_files.has_key(oid):
             handle, name = tempfile.mkstemp()
-            handle.close()
+            os.close(handle)
             self.blob_files[oid] = name
         return self.blob_files[oid]