Commit d6919751 authored by Christian Theune's avatar Christian Theune

Fixed bug #130459 by providing a separate well-known temporary directory where

uncommitted data is placed.
parent ecd38de5
...@@ -83,6 +83,8 @@ Transactions ...@@ -83,6 +83,8 @@ Transactions
Blobs Blobs
----- -----
- (3.8b5) Fixed bug #130459: Packing was broken by uncommitted blob data.
- (3.8b4) Fixed bug #127182: Blobs were subclassable which was not desired. - (3.8b4) Fixed bug #127182: Blobs were subclassable which was not desired.
- (3.8b3) Fixed bug #126007: tpc_abort had untested code path that was - (3.8b3) Fixed bug #126007: tpc_abort had untested code path that was
......
...@@ -244,7 +244,7 @@ class Blob(persistent.Persistent): ...@@ -244,7 +244,7 @@ class Blob(persistent.Persistent):
def cleanup(ref): def cleanup(ref):
if os.path.exists(filename): if os.path.exists(filename):
os.remove(filename) os.remove(filename)
self._p_blob_ref = weakref.ref(self, cleanup) self._p_blob_ref = weakref.ref(self, cleanup)
return filename return filename
...@@ -294,6 +294,7 @@ class FilesystemHelper: ...@@ -294,6 +294,7 @@ class FilesystemHelper:
def __init__(self, base_dir): def __init__(self, base_dir):
self.base_dir = base_dir self.base_dir = base_dir
self.temp_dir = os.path.join(base_dir, 'tmp')
def create(self): def create(self):
if not os.path.exists(self.base_dir): if not os.path.exists(self.base_dir):
...@@ -301,6 +302,11 @@ class FilesystemHelper: ...@@ -301,6 +302,11 @@ class FilesystemHelper:
log("Blob cache directory '%s' does not exist. " log("Blob cache directory '%s' does not exist. "
"Created new directory." % self.base_dir, "Created new directory." % self.base_dir,
level=logging.INFO) level=logging.INFO)
if not os.path.exists(self.temp_dir):
os.makedirs(self.temp_dir, 0700)
log("Blob temporary directory '%s' does not exist. "
"Created new directory." % self.temp_dir,
level=logging.INFO)
def isSecure(self, path): def isSecure(self, path):
"""Ensure that (POSIX) path mode bits are 0700.""" """Ensure that (POSIX) path mode bits are 0700."""
...@@ -375,6 +381,17 @@ class FilesystemHelper: ...@@ -375,6 +381,17 @@ class FilesystemHelper:
oids.append(oid) oids.append(oid)
return oids return oids
def listOIDs(self):
"""Lists all OIDs and their paths.
"""
for candidate in os.listdir(self.base_dir):
if candidate == 'tmp':
continue
oid = utils.repr_to_oid(candidate)
yield oid, self.getPathForOID(oid)
class BlobStorage(SpecificationDecoratorBase): class BlobStorage(SpecificationDecoratorBase):
"""A storage to support blobs.""" """A storage to support blobs."""
...@@ -404,8 +421,7 @@ class BlobStorage(SpecificationDecoratorBase): ...@@ -404,8 +421,7 @@ class BlobStorage(SpecificationDecoratorBase):
@non_overridable @non_overridable
def temporaryDirectory(self): def temporaryDirectory(self):
return self.fshelper.base_dir return self.fshelper.temp_dir
@non_overridable @non_overridable
def __repr__(self): def __repr__(self):
...@@ -471,18 +487,8 @@ class BlobStorage(SpecificationDecoratorBase): ...@@ -471,18 +487,8 @@ class BlobStorage(SpecificationDecoratorBase):
# if they are still needed by attempting to load the revision # if they are still needed by attempting to load the revision
# of that object from the database. This is maybe the slowest # of that object from the database. This is maybe the slowest
# possible way to do this, but it's safe. # possible way to do this, but it's safe.
# XXX we should be tolerant of "garbage" directories/files in
# the base_directory here.
# XXX If this method gets refactored we have to watch out for extra
# files from uncommitted transactions. The current implementation
# doesn't have a problem, but future refactorings likely will.
base_dir = self.fshelper.base_dir base_dir = self.fshelper.base_dir
for oid_repr in os.listdir(base_dir): for oid, oid_path in self.fshelper.listOIDs():
oid = utils.repr_to_oid(oid_repr)
oid_path = os.path.join(base_dir, oid_repr)
files = os.listdir(oid_path) files = os.listdir(oid_path)
files.sort() files.sort()
...@@ -501,11 +507,8 @@ class BlobStorage(SpecificationDecoratorBase): ...@@ -501,11 +507,8 @@ class BlobStorage(SpecificationDecoratorBase):
@non_overridable @non_overridable
def _packNonUndoing(self, packtime, referencesf): def _packNonUndoing(self, packtime, referencesf):
base_dir = self.fshelper.base_dir base_dir = self.fshelper.base_dir
for oid_repr in os.listdir(base_dir): for oid, oid_path in self.fshelper.listOIDs():
oid = utils.repr_to_oid(oid_repr)
oid_path = os.path.join(base_dir, oid_repr)
exists = True exists = True
try: try:
self.load(oid, None) # no version support self.load(oid, None) # no version support
except (POSKeyError, KeyError): except (POSKeyError, KeyError):
......
...@@ -29,18 +29,9 @@ Set up: ...@@ -29,18 +29,9 @@ Set up:
>>> storagefile = mktemp() >>> storagefile = mktemp()
>>> blob_dir = mkdtemp() >>> blob_dir = mkdtemp()
A helper method to assure a unique timestamp across multiple platforms. This A helper method to assure a unique timestamp across multiple platforms:
method also makes sure that after retrieving a timestamp that was *before* a
transaction was committed, that at least one second passes so the packing time >>> from ZODB.tests.testblob import new_time
actually is before the commit time.
>>> import time
>>> def new_time():
... now = new_time = time.time()
... while new_time <= now:
... new_time = time.time()
... time.sleep(1)
... return new_time
UNDOING UNDOING
======= =======
...@@ -170,7 +161,7 @@ We need an database with a NON-undoing blob supporting storage: ...@@ -170,7 +161,7 @@ We need an database with a NON-undoing blob supporting storage:
>>> base_storage = MappingStorage('storage') >>> base_storage = MappingStorage('storage')
>>> blob_storage = BlobStorage(blob_dir, base_storage) >>> blob_storage = BlobStorage(blob_dir, base_storage)
>>> database = DB(blob_storage) >>> database = DB(blob_storage)
Create our root object: Create our root object:
>>> connection1 = database.open() >>> connection1 = database.open()
...@@ -228,7 +219,7 @@ Do a pack to the slightly before the first revision was written: ...@@ -228,7 +219,7 @@ Do a pack to the slightly before the first revision was written:
>>> blob_storage.pack(packtime, referencesf) >>> blob_storage.pack(packtime, referencesf)
>>> [ os.path.exists(x) for x in fns ] >>> [ os.path.exists(x) for x in fns ]
[False, False, False, False, True] [False, False, False, False, True]
Do a pack to now: Do a pack to now:
>>> packtime = new_time() >>> packtime = new_time()
......
...@@ -31,17 +31,18 @@ First, we need a datatabase with blob support:: ...@@ -31,17 +31,18 @@ First, we need a datatabase with blob support::
>>> from ZODB.blob import BlobStorage >>> from ZODB.blob import BlobStorage
>>> from ZODB.DB import DB >>> from ZODB.DB import DB
>>> from tempfile import mkdtemp >>> from tempfile import mkdtemp
>>> import os.path
>>> base_storage = MappingStorage("test") >>> base_storage = MappingStorage("test")
>>> blob_dir = mkdtemp() >>> blob_dir = mkdtemp()
>>> blob_storage = BlobStorage(blob_dir, base_storage) >>> blob_storage = BlobStorage(blob_dir, base_storage)
>>> database = DB(blob_storage) >>> database = DB(blob_storage)
Now we create a blob and put it in the database. After that we open it for Now we create a blob and put it in the database. After that we open it for
writing and expect the file to be in the blob directory:: writing and expect the file to be in the blob temporary directory::
>>> blob = Blob() >>> blob = Blob()
>>> connection = database.open() >>> connection = database.open()
>>> connection.add(blob) >>> connection.add(blob)
>>> w = blob.open('w') >>> w = blob.open('w')
>>> w.name.startswith(blob_dir) >>> w.name.startswith(os.path.join(blob_dir, 'tmp'))
True True
...@@ -13,6 +13,7 @@ ...@@ -13,6 +13,7 @@
############################################################################## ##############################################################################
import base64, os, shutil, tempfile, unittest import base64, os, shutil, tempfile, unittest
import time
from zope.testing import doctest from zope.testing import doctest
import ZODB.tests.util import ZODB.tests.util
...@@ -26,6 +27,22 @@ import transaction ...@@ -26,6 +27,22 @@ import transaction
from ZODB.tests.testConfig import ConfigTestBase from ZODB.tests.testConfig import ConfigTestBase
from ZConfig import ConfigurationSyntaxError from ZConfig import ConfigurationSyntaxError
def new_time():
"""Create a _new_ time stamp.
This method also makes sure that after retrieving a timestamp that was
*before* a transaction was committed, that at least one second passes so
the packing time actually is before the commit time.
"""
now = new_time = time.time()
while new_time <= now:
new_time = time.time()
time.sleep(1)
return new_time
class BlobConfigTestBase(ConfigTestBase): class BlobConfigTestBase(ConfigTestBase):
def setUp(self): def setUp(self):
...@@ -284,7 +301,7 @@ Works with savepoints too: ...@@ -284,7 +301,7 @@ Works with savepoints too:
>>> root['blob2'].open().read() >>> root['blob2'].open().read()
'test2' 'test2'
>>> os.rename = os_rename >>> os.rename = os_rename
>>> logger.propagate = True >>> logger.propagate = True
>>> logger.setLevel(0) >>> logger.setLevel(0)
...@@ -292,6 +309,86 @@ Works with savepoints too: ...@@ -292,6 +309,86 @@ Works with savepoints too:
""" """
def packing_with_uncommitted_data_non_undoing():
"""
This covers regression for bug #130459.
When uncommitted data exists it formerly was written to the root of the
blob_directory and confused our packing strategy. We now use a separate
temporary directory that is ignored while packing.
>>> import transaction
>>> from ZODB.MappingStorage import MappingStorage
>>> from ZODB.blob import BlobStorage
>>> from ZODB.DB import DB
>>> from ZODB.serialize import referencesf
>>> from tempfile import mkdtemp
>>> base_storage = MappingStorage("test")
>>> blob_dir = mkdtemp()
>>> blob_storage = BlobStorage(blob_dir, base_storage)
>>> database = DB(blob_storage)
>>> connection = database.open()
>>> root = connection.root()
>>> from ZODB.blob import Blob
>>> root['blob'] = Blob()
>>> connection.add(root['blob'])
>>> root['blob'].open('w').write('test')
>>> blob_storage.pack(new_time(), referencesf)
Clean up:
>>> database.close()
>>> import shutil
>>> shutil.rmtree(blob_dir)
"""
def packing_with_uncommitted_data_undoing():
"""
This covers regression for bug #130459.
When uncommitted data exists it formerly was written to the root of the
blob_directory and confused our packing strategy. We now use a separate
temporary directory that is ignored while packing.
>>> import transaction
>>> from ZODB.FileStorage.FileStorage import FileStorage
>>> from ZODB.blob import BlobStorage
>>> from ZODB.DB import DB
>>> from ZODB.serialize import referencesf
>>> from tempfile import mkdtemp, mktemp
>>> storagefile = mktemp()
>>> base_storage = FileStorage(storagefile)
>>> blob_dir = mkdtemp()
>>> blob_storage = BlobStorage(blob_dir, base_storage)
>>> database = DB(blob_storage)
>>> connection = database.open()
>>> root = connection.root()
>>> from ZODB.blob import Blob
>>> root['blob'] = Blob()
>>> connection.add(root['blob'])
>>> root['blob'].open('w').write('test')
>>> blob_storage.pack(new_time(), referencesf)
Clean up:
>>> database.close()
>>> import shutil
>>> shutil.rmtree(blob_dir)
>>> os.unlink(storagefile)
>>> os.unlink(storagefile+".index")
>>> os.unlink(storagefile+".tmp")
"""
def test_suite(): def test_suite():
suite = unittest.TestSuite() suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(ZODBBlobConfigTest)) suite.addTest(unittest.makeSuite(ZODBBlobConfigTest))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment