Commit 5d341e3c authored by Jim Fulton's avatar Jim Fulton

Merged tseaver-lp143158-feature, which adds the new feature:

- Added a '--kill-old-on-full' argument to the backup options:  if passed,
  remove any older full or incremental backup files from the repository after
  doing a full backup. (https://bugs.launchpad.net/zope2/+bug/143158)
parent f32b1627
...@@ -5,6 +5,18 @@ ...@@ -5,6 +5,18 @@
3.10.0a2 (2010-??-??) 3.10.0a2 (2010-??-??)
===================== =====================
New Features
------------
- Added a '--kill-old-on-full' argument to the backup options: if passed,
remove any older full or incremental backup files from the repository after
doing a full backup. (https://bugs.launchpad.net/zope2/+bug/143158)
- When transactions are aborted, new object ids allocated during the
transaction are saved and used in subsequent transactions. This can
help in situations where object ids are used as BTree keys and the
sequential allocation of object ids leads to conflict errors.
Bugs Fixed Bugs Fixed
---------- ----------
...@@ -14,14 +26,6 @@ Bugs Fixed ...@@ -14,14 +26,6 @@ Bugs Fixed
- When using using a ClientStorage in a Storage server, there was a - When using using a ClientStorage in a Storage server, there was a
threading bug that caused clients to get disconnected. threading bug that caused clients to get disconnected.
New Features
------------
- When transactions are aborted, new object ids allocated during the
transaction are saved and used in subsequent transactions. This can
help in situations where object ids are used as BTree keys and the
sequential allocation of object ids leads to conflict errors.
3.10.0a1 (2010-02-08) 3.10.0a1 (2010-02-08)
===================== =====================
......
...@@ -50,6 +50,11 @@ Options for -B/--backup: ...@@ -50,6 +50,11 @@ Options for -B/--backup:
Compress with gzip the backup files. Uses the default zlib Compress with gzip the backup files. Uses the default zlib
compression level. By default, gzip compression is not used. compression level. By default, gzip compression is not used.
-k / --kill-old-on-full
If a full backup is created, remove any prior full or incremental
backup files (and associated metadata files) from the repository
directory.
Options for -R/--recover: Options for -R/--recover:
-D str -D str
--date=str --date=str
...@@ -109,10 +114,20 @@ def log(msg, *args): ...@@ -109,10 +114,20 @@ def log(msg, *args):
def parseargs(argv): def parseargs(argv):
global VERBOSE global VERBOSE
try: try:
opts, args = getopt.getopt(argv, 'BRvhf:r:FD:o:Qz', opts, args = getopt.getopt(argv, 'BRvhr:f:FQzkD:o:',
['backup', 'recover', 'verbose', 'help', ['backup',
'file=', 'repository=', 'full', 'date=', 'recover',
'output=', 'quick', 'gzip']) 'verbose',
'help',
'repository=',
'file=',
'full',
'quick',
'gzip',
'kill-old-on-full',
'date=',
'output=',
])
except getopt.error, msg: except getopt.error, msg:
usage(1, msg) usage(1, msg)
...@@ -125,6 +140,7 @@ def parseargs(argv): ...@@ -125,6 +140,7 @@ def parseargs(argv):
output = None # where to write recovered data; None = stdout output = None # where to write recovered data; None = stdout
quick = False # -Q flag state quick = False # -Q flag state
gzip = False # -z flag state gzip = False # -z flag state
killold = False # -k flag state
options = Options() options = Options()
...@@ -155,6 +171,8 @@ def parseargs(argv): ...@@ -155,6 +171,8 @@ def parseargs(argv):
options.output = arg options.output = arg
elif opt in ('-z', '--gzip'): elif opt in ('-z', '--gzip'):
options.gzip = True options.gzip = True
elif opt in ('-k', '--kill-old-on-full'):
options.killold = True
else: else:
assert False, (opt, arg) assert False, (opt, arg)
...@@ -179,6 +197,9 @@ def parseargs(argv): ...@@ -179,6 +197,9 @@ def parseargs(argv):
if options.file is not None: if options.file is not None:
log('--file option is ignored in recover mode') log('--file option is ignored in recover mode')
options.file = None options.file = None
if options.killold is not None:
log('--kill-old-on-full option is ignored in recover mode')
options.killold = None
return options return options
...@@ -351,6 +372,39 @@ def scandat(repofiles): ...@@ -351,6 +372,39 @@ def scandat(repofiles):
return fn, startpos, endpos, sum return fn, startpos, endpos, sum
def delete_old_backups(options):
# Delete all full backup files except for the most recent full backup file
all = filter(is_data_file, os.listdir(options.repository))
all.sort()
deletable = []
full = []
for fname in all:
root, ext = os.path.splitext(fname)
if ext in ('.fs', '.fsz'):
full.append(fname)
if ext in ('.fs', '.fsz', '.deltafs', '.deltafsz'):
deletable.append(fname)
# keep most recent full
if not full:
return
recentfull = full.pop(-1)
deletable.remove(recentfull)
root, ext = os.path.splitext(recentfull)
dat = root + '.dat'
if dat in deletable:
deletable.remove(dat)
for fname in deletable:
log('removing old backup file %s (and .dat)', fname)
root, ext = os.path.splitext(fname)
try:
os.unlink(os.path.join(options.repository, root + '.dat'))
except OSError:
pass
os.unlink(os.path.join(options.repository, fname))
def do_full_backup(options): def do_full_backup(options):
# Find the file position of the last completed transaction. # Find the file position of the last completed transaction.
...@@ -376,6 +430,8 @@ def do_full_backup(options): ...@@ -376,6 +430,8 @@ def do_full_backup(options):
fp.flush() fp.flush()
os.fsync(fp.fileno()) os.fsync(fp.fileno())
fp.close() fp.close()
if options.killold:
delete_old_backups(options)
def do_incremental_backup(options, reposz, repofiles): def do_incremental_backup(options, reposz, repofiles):
......
...@@ -168,8 +168,94 @@ class RepozoTests(unittest.TestCase): ...@@ -168,8 +168,94 @@ class RepozoTests(unittest.TestCase):
(correctpath, when, ' '.join(argv))) (correctpath, when, ' '.join(argv)))
self.assertEquals(fguts, gguts, msg) self.assertEquals(fguts, gguts, msg)
class Test_delete_old_backups(unittest.TestCase):
_repository_directory = None
def tearDown(self):
if self._repository_directory is not None:
from shutil import rmtree
rmtree(self._repository_directory)
def _callFUT(self, options=None, filenames=()):
from ZODB.scripts.repozo import delete_old_backups
if options is None:
options = self._makeOptions(filenames)
delete_old_backups(options)
def _makeOptions(self, filenames=()):
import tempfile
dir = self._repository_directory = tempfile.mkdtemp()
for filename in filenames:
fqn = os.path.join(dir, filename)
f = open(fqn, 'wb')
f.write('testing delete_old_backups')
f.close()
class Options(object):
repository = dir
return Options()
def test_empty_dir_doesnt_raise(self):
self._callFUT()
self.assertEqual(len(os.listdir(self._repository_directory)), 0)
def test_no_repozo_files_doesnt_raise(self):
FILENAMES = ['bogus.txt', 'not_a_repozo_file']
self._callFUT(filenames=FILENAMES)
remaining = os.listdir(self._repository_directory)
self.assertEqual(len(remaining), len(FILENAMES))
for name in FILENAMES:
fqn = os.path.join(self._repository_directory, name)
self.failUnless(os.path.isfile(fqn))
def test_doesnt_remove_current_repozo_files(self):
FILENAMES = ['2009-12-20-10-08-03.fs', '2009-12-20-10-08-03.dat']
self._callFUT(filenames=FILENAMES)
remaining = os.listdir(self._repository_directory)
self.assertEqual(len(remaining), len(FILENAMES))
for name in FILENAMES:
fqn = os.path.join(self._repository_directory, name)
self.failUnless(os.path.isfile(fqn))
def test_removes_older_repozo_files(self):
OLDER_FULL = ['2009-12-20-00-01-03.fs', '2009-12-20-00-01-03.dat']
DELTAS = ['2009-12-21-00-00-01.deltafs', '2009-12-22-00-00-01.deltafs']
CURRENT_FULL = ['2009-12-23-00-00-01.fs', '2009-12-23-00-00-01.dat']
FILENAMES = OLDER_FULL + DELTAS + CURRENT_FULL
self._callFUT(filenames=FILENAMES)
remaining = os.listdir(self._repository_directory)
self.assertEqual(len(remaining), len(CURRENT_FULL))
for name in OLDER_FULL:
fqn = os.path.join(self._repository_directory, name)
self.failIf(os.path.isfile(fqn))
for name in DELTAS:
fqn = os.path.join(self._repository_directory, name)
self.failIf(os.path.isfile(fqn))
for name in CURRENT_FULL:
fqn = os.path.join(self._repository_directory, name)
self.failUnless(os.path.isfile(fqn))
def test_removes_older_repozo_files_zipped(self):
OLDER_FULL = ['2009-12-20-00-01-03.fsz', '2009-12-20-00-01-03.dat']
DELTAS = ['2009-12-21-00-00-01.deltafsz',
'2009-12-22-00-00-01.deltafsz']
CURRENT_FULL = ['2009-12-23-00-00-01.fsz', '2009-12-23-00-00-01.dat']
FILENAMES = OLDER_FULL + DELTAS + CURRENT_FULL
self._callFUT(filenames=FILENAMES)
remaining = os.listdir(self._repository_directory)
self.assertEqual(len(remaining), len(CURRENT_FULL))
for name in OLDER_FULL:
fqn = os.path.join(self._repository_directory, name)
self.failIf(os.path.isfile(fqn))
for name in DELTAS:
fqn = os.path.join(self._repository_directory, name)
self.failIf(os.path.isfile(fqn))
for name in CURRENT_FULL:
fqn = os.path.join(self._repository_directory, name)
self.failUnless(os.path.isfile(fqn))
def test_suite(): def test_suite():
return unittest.TestSuite([ return unittest.TestSuite([
unittest.makeSuite(RepozoTests), unittest.makeSuite(RepozoTests),
unittest.makeSuite(Test_delete_old_backups),
]) ])
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment