Commit 7f051682 authored by Jim Fulton's avatar Jim Fulton

- File-storage pack clean-up tasks that can take a long time

  unnecessarily blocked other activity.
parent 65558dbf
...@@ -13,6 +13,9 @@ Bugs Fixed ...@@ -13,6 +13,9 @@ Bugs Fixed
implemented daemon behavior by forking. Now, the client thread implemented daemon behavior by forking. Now, the client thread
isn't created until needed. isn't created until needed.
- File-storage pack clean-up tasks that can take a long time
unnecessarily blocked other activity.
3.9.1 (2009-10-01) 3.9.1 (2009-10-01)
================== ==================
......
...@@ -1117,6 +1117,8 @@ class FileStorage( ...@@ -1117,6 +1117,8 @@ class FileStorage(
if self.blob_dir and os.path.exists(self.blob_dir + ".old"): if self.blob_dir and os.path.exists(self.blob_dir + ".old"):
ZODB.blob.remove_committed_dir(self.blob_dir + ".old") ZODB.blob.remove_committed_dir(self.blob_dir + ".old")
cleanup = []
have_commit_lock = False have_commit_lock = False
try: try:
pack_result = None pack_result = None
...@@ -1139,19 +1141,20 @@ class FileStorage( ...@@ -1139,19 +1141,20 @@ class FileStorage(
# OK, we're beyond the point of no return # OK, we're beyond the point of no return
os.rename(self._file_name + '.pack', self._file_name) os.rename(self._file_name + '.pack', self._file_name)
if not self.pack_keep_old:
os.remove(oldpath)
self._file = open(self._file_name, 'r+b') self._file = open(self._file_name, 'r+b')
self._initIndex(index, self._tindex) self._initIndex(index, self._tindex)
self._pos = opos self._pos = opos
self._save_index() finally:
self._lock_release()
# We're basically done. Now we need to deal with removed
# blobs and removing the .old file (see further down).
if self.blob_dir: if self.blob_dir:
self._commit_lock_release() self._commit_lock_release()
have_commit_lock = False have_commit_lock = False
self._remove_blob_files_tagged_for_removal_during_pack() self._remove_blob_files_tagged_for_removal_during_pack()
finally:
self._lock_release()
finally: finally:
if have_commit_lock: if have_commit_lock:
self._commit_lock_release() self._commit_lock_release()
...@@ -1159,6 +1162,14 @@ class FileStorage( ...@@ -1159,6 +1162,14 @@ class FileStorage(
self._pack_is_in_progress = False self._pack_is_in_progress = False
self._lock_release() self._lock_release()
if not self.pack_keep_old:
os.remove(oldpath)
self._lock_acquire()
try:
self._save_index()
finally:
self._lock_release()
def _remove_blob_files_tagged_for_removal_during_pack(self): def _remove_blob_files_tagged_for_removal_during_pack(self):
lblob_dir = len(self.blob_dir) lblob_dir = len(self.blob_dir)
...@@ -1167,13 +1178,38 @@ class FileStorage( ...@@ -1167,13 +1178,38 @@ class FileStorage(
link_or_copy = ZODB.blob.link_or_copy link_or_copy = ZODB.blob.link_or_copy
# Helper to clean up dirs left empty after moving things to old # Helper to clean up dirs left empty after moving things to old
def maybe_remove_empty_dir_containing(path): def maybe_remove_empty_dir_containing(path, level=0):
path = os.path.dirname(path) path = os.path.dirname(path)
if len(path) <= lblob_dir: if len(path) <= lblob_dir or os.listdir(path):
return return
# Path points to an empty dir. There may be a race. We
# might have just removed the dir for an oid (or a parent
# dir) and while we're cleaning up it's parent, another
# thread is adding a new entry to it.
# We don't have to worry about level 0, as this is just a
# directory containing an object's revisions. If it is
# enmpty, the object must have been garbage.
# If the level is 1 or higher, we need to be more
# careful. We'll get the storage lock and double check
# that the dir is still empty before removing it.
removed = False
if level:
self._lock_acquire()
try:
if not os.listdir(path): if not os.listdir(path):
os.rmdir(path) os.rmdir(path)
maybe_remove_empty_dir_containing(path) removed = True
finally:
if level:
self._lock_release()
if removed:
maybe_remove_empty_dir_containing(path, level+1)
if self.pack_keep_old: if self.pack_keep_old:
# Helpers that move oid dir or revision file to the old dir. # Helpers that move oid dir or revision file to the old dir.
...@@ -1203,7 +1239,7 @@ class FileStorage( ...@@ -1203,7 +1239,7 @@ class FileStorage(
# Hm, already gone. Odd. # Hm, already gone. Odd.
continue continue
handle_dir(path) handle_dir(path)
maybe_remove_empty_dir_containing(path) maybe_remove_empty_dir_containing(path, 1)
continue continue
if len(line) != 16: if len(line) != 16:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment