Added --list-increment-sizes option

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@336 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

trunk/rdiff-backup/CHANGELOG

+New in v0.13.0 (2003/07/02)
+---------------------------
+
+To prevent the buildup of confusing and error-prone options, the
+capabilities of the source and destination file systems are now
+autodetected.  Detected features include allowed characters, extended
+attributes, access control lists, hard links, ownership, and directory
+fsyncing.  Options such as --windows-mode, --chars-to-quote,
+--quoting-char, and --windows-restore-mode have been removed.
+
+Now rdiff-backup supports extended attributes.  To take advantage of
+this you will need the python module pyxattr and a file system that
+supports EAs.  Thanks to Greg Freemyer for discussion.
+
+Added --list-increment-sizes switch, which tells you how much space
+the various backup files take up.  (Suggested by Andrew Bressen)
+
+
 New in v0.12.0 (2003/06/26)
 ---------------------------
 

trunk/rdiff-backup/rdiff-backup.1

@@ -11,7 +11,8 @@ rdiff-backup \- local/remote mirror and incremental backup
 .B {{ -l | --list-increments }
 .BI "| --remove-older-than " time_interval
 .BI "| --list-at-time " time
-.BI "| --list-changed-since " time }
+.BI "| --list-changed-since " time
+.B "| --list-increment-sizes "}
 .BI [[[ user@ ] host2.foo ]:: destination_directory ]
 
 .B rdiff-backup --calculate-average
@@ -186,6 +187,13 @@ List the number and date of partial incremental backups contained in
 the specified destination directory.  No backup or restore will take
 place if this option is given.
 .TP
+.B --list-increment-sizes
+List the total size of all the increment and mirror files by time.
+This may be helpful in deciding how many increments to keep, and when
+to --remove-older-than.  Specifying a subdirectory is allowable; then
+only the sizes of the mirror and increments pertaining to that
+subdirectory will be listed.
+.TP
 .B "-m, --mirror-only"
 Do not create an rdiff-backup-data directory or make any increments.
 In this mode rdiff-backup is similar to rsync (but usually

trunk/rdiff-backup/rdiff_backup/Main.py

@@ -55,9 +55,9 @@ def parse_cmdlineoptions(arglist):
 		  "include-filelist=", "include-filelist-stdin",
 		  "include-globbing-filelist=", "include-regexp=",
 		  "list-at-time=", "list-changed-since=", "list-increments",
-		  "no-compare-inode", "no-compression",
-		  "no-compression-regexp=", "no-file-statistics",
-		  "no-hard-links", "null-separator",
+		  "list-increment-sizes", "no-compare-inode",
+		  "no-compression", "no-compression-regexp=",
+		  "no-file-statistics", "no-hard-links", "null-separator",
 		  "override-chars-to-quote=", "parsable-output",
 		  "print-statistics", "remote-cmd=", "remote-schema=",
 		  "remove-older-than=", "restore-as-of=", "restrict=",
@@ -105,6 +105,7 @@ def parse_cmdlineoptions(arglist):
 			restore_timestr, action = arg, "list-changed-since"
 		elif opt == "-l" or opt == "--list-increments":
 			action = "list-increments"
+		elif opt == '--list-increment-sizes': action = 'list-increment-sizes'
 		elif opt == "--no-compare-inode": Globals.set("compare_inode", 0)
 		elif opt == "--no-compression": Globals.set("compression", None)
 		elif opt == "--no-compression-regexp":
@@ -147,9 +148,9 @@ def check_action():
 	"""Check to make sure action is compatible with args"""
 	global action
 	arg_action_dict = {0: ['server'],
-					   1: ['list-increments', 'remove-older-than',
-						   'list-at-time', 'list-changed-since',
-						   'check-destination-dir'],
+					   1: ['list-increments', 'list-increment-sizes',
+						   'remove-older-than', 'list-at-time',
+						   'list-changed-since', 'check-destination-dir'],
 					   2: ['backup', 'restore', 'restore-as-of']}
 	l = len(args)
 	if not action: assert l == 2, args # cannot tell backup or restore yet
@@ -163,7 +164,7 @@ def final_set_action(rps):
 	global action
 	if action: return
 	assert len(rps) == 2, rps
-	if restore_get_root(rps[0]): action = "restore"
+	if restore_set_root(rps[0]): action = "restore"
 	else: action = "backup"
 
 def commandline_error(message):
@@ -194,6 +195,7 @@ def take_action(rps):
 	elif action == "list-at-time": ListAtTime(rps[0])
 	elif action == "list-changed-since": ListChangedSince(rps[0])
 	elif action == "list-increments": ListIncrements(rps[0])
+	elif action == 'list-increment-sizes': ListIncrementSizes(rps[0])
 	elif action == "remove-older-than": RemoveOlderThan(rps[0])
 	elif action == "calculate-average": CalculateAverage(rps)
 	elif action == "check-destination-dir": CheckDest(rps[0])
@@ -373,7 +375,7 @@ def Restore(src_rp, dest_rp, restore_as_of = None):
 	mirror file), dest_rp should be the target rp to be written.
 
 	"""
-	if not restore_root_set: assert restore_get_root(src_rp)
+	if not restore_root_set: assert restore_set_root(src_rp)
 	restore_check_paths(src_rp, dest_rp, restore_as_of)
 	restore_set_fs_globals(Globals.rbdir)
 	src_rp = restore_init_quoting(src_rp)
@@ -449,7 +451,7 @@ def restore_check_paths(rpin, rpout, restoreasof = None):
 		Log.FatalError("Restore target %s already exists, "
 					   "specify --force to overwrite." % rpout.path)
 
-def restore_check_backup_dir(mirror_root, src_rp, restore_as_of):
+def restore_check_backup_dir(mirror_root, src_rp = None, restore_as_of = 1):
 	"""Make sure backup dir root rpin is in consistent state"""
 	if not restore_as_of and not src_rp.isincfile():
 		Log.FatalError("""File %s does not look like an increment file.
@@ -466,7 +468,7 @@ Try restoring from an increment file (the filenames look like
 		"rdiff-with --check-destination-dir option to revert directory "
 		"to state before unsuccessful session." % (mirror_root.path,))
 
-def restore_get_root(rpin):
+def restore_set_root(rpin):
 	"""Set data dir, restore_root and index, or return None if fail
 
 	The idea here is to keep backing up on the path until we find
@@ -480,7 +482,7 @@ def restore_get_root(rpin):
 	funny way, using symlinks or somesuch.
 
 	"""
-	global restore_root, restore_index
+	global restore_root, restore_index, restore_root_set
 	if rpin.isincfile(): relpath = rpin.getincbase().path
 	else: relpath = rpin.path
 	pathcomps = os.path.join(rpin.conn.os.getcwd(), relpath).split("/")
@@ -510,15 +512,16 @@ def restore_get_root(rpin):
 				(len(from_datadir) == 2 and
 				 from_datadir[1].startswith('increments'))), from_datadir
 		restore_index = from_datadir[2:]
+	restore_root_set = 1
 	return 1
 
 
 def ListIncrements(rp):
 	"""Print out a summary of the increments and their times"""
-	mirror_root, index = restore_get_root(rp)
-	restore_check_backup_dir(mirror_root)
-	mirror_rp = mirror_root.new_index(index)
-	inc_rpath = Globals.rbdir.append_path('increments', index)
+	assert restore_set_root(rp)
+	restore_check_backup_dir(restore_root)
+	mirror_rp = restore_root.new_index(restore_index)
+	inc_rpath = Globals.rbdir.append_path('increments', restore_index)
 	incs = restore.get_inclist(inc_rpath)
 	mirror_time = restore.MirrorStruct.get_mirror_time()
 	if Globals.parsable_output:
@@ -526,6 +529,13 @@ def ListIncrements(rp):
 	else: print manage.describe_incs_human(incs, mirror_time, mirror_rp)
 
 
+def ListIncrementSizes(rp):
+	"""Print out a summary of the increments """
+	assert restore_set_root(rp)
+	restore_check_backup_dir(restore_root)
+	print manage.ListIncrementSizes(restore_root, restore_index)
+
+
 def CalculateAverage(rps):
 	"""Print out the average of the given statistics files"""
 	statobjs = map(lambda rp: StatsObj().read_stats_from_rp(rp), rps)
@@ -575,10 +585,10 @@ def ListChangedSince(rp):
 	"""List all the files under rp that have changed since restoretime"""
 	try: rest_time = Time.genstrtotime(restore_timestr)
 	except Time.TimeException, exc: Log.FatalError(str(exc))
-	mirror_root, index = restore_get_root(rp)
-	restore_check_backup_dir(mirror_root)
-	mirror_rp = mirror_root.new_index(index)
-	inc_rp = mirror_rp.append_path("increments", index)
+	assert restore_set_root(rp)
+	restore_check_backup_dir(restore_root)
+	mirror_rp = restore_root.new_index(restore_index)
+	inc_rp = mirror_rp.append_path("increments", restore_index)
 	restore.ListChangedSince(mirror_rp, inc_rp, rest_time)
 
 
@@ -586,10 +596,10 @@ def ListAtTime(rp):
 	"""List files in archive under rp that are present at restoretime"""
 	try: rest_time = Time.genstrtotime(restore_timestr)
 	except Time.TimeException, exc: Log.FatalError(str(exc))
-	mirror_root, index = restore_get_root(rp)
-	restore_check_backup_dir(mirror_root)
-	mirror_rp = mirror_root.new_index(index)
-	inc_rp = mirror_rp.append_path("increments", index)
+	assert restore_set_root(rp)
+	restore_check_backup_dir(restore_root)
+	mirror_rp = restore_root.new_index(restore_index)
+	inc_rp = mirror_rp.append_path("increments", restore_index)
 	restore.ListAtTime(mirror_rp, inc_rp, rest_time)
 	
 

trunk/rdiff-backup/rdiff_backup/Security.py

@@ -76,7 +76,7 @@ def set_security_level(action, cmdpairs):
 			rdir = tempfile.gettempdir()
 		elif islocal(cp1):
 			sec_level = "read-only"
-			Main.restore_get_root(rpath.RPath(Globals.local_connection,
+			Main.restore_set_root(rpath.RPath(Globals.local_connection,
 											  getpath(cp1)))
 			rdir = Main.restore_root.path
 		else:
@@ -94,9 +94,9 @@ def set_security_level(action, cmdpairs):
 			assert islocal(cp2)
 			sec_level = "all"
 			rdir = getpath(cp2)
-	elif (action == "test-server" or action == "list-increments" or
-		  action == "list-at-time" or action == "list-changed-since"
-		  or action == "calculate-average" or action == "remove-older-than"):
+	elif action in ["test-server", "list-increments", 'list-increment-sizes',
+					 "list-at-time", "list-changed-since",
+					 "calculate-average", "remove-older-than"]:
 		sec_level = "minimal"
 		rdir = tempfile.gettempdir()
 	else: assert 0, "Unknown action %s" % action

trunk/rdiff-backup/rdiff_backup/manage.py

@@ -21,7 +21,7 @@
 
 from __future__ import generators
 from log import Log
-import Globals, Time, static, manage
+import Globals, Time, static, manage, statistics, restore, selection
 
 
 class ManageException(Exception): pass
@@ -128,3 +128,77 @@ class IncObj:
 		s = ["Increment file %s" % self.incrp.path,
 			 "Date: %s" % self.pretty_time()]
 		return "\n".join(s)
+
+
+def ListIncrementSizes(mirror_root, index):
+	"""Return string summarizing the size of all the increments"""
+	stat_obj = statistics.StatsObj() # used for byte summary string
+	def get_total(rp_iter):
+		"""Return the total size of everything in rp_iter"""
+		total = 0
+		for rp in rp_iter: total += rp.getsize()
+		return total
+
+	def get_time_dict(inc_iter):
+		"""Return dictionary pairing times to total size of incs"""
+		time_dict = {}
+		for inc in inc_iter:
+			if not inc.isincfile(): continue
+			t = inc.getinctime()
+			if not time_dict.has_key(t): time_dict[t] = 0
+			time_dict[t] += inc.getsize()
+		return time_dict
+
+	def get_mirror_select():
+		"""Return iterator of mirror rpaths"""
+		mirror_base = mirror_root.new_index(index)
+		mirror_select = selection.Select(mirror_base)
+		if not index: # must exclude rdiff-backup-directory
+			mirror_select.parse_rbdir_exclude()
+		return mirror_select.set_iter()
+
+	def get_inc_select():
+		"""Return iterator of increment rpaths"""
+		inc_base = Globals.rbdir.append_path('increments', index)
+		for base_inc in restore.get_inclist(inc_base): yield base_inc
+		if inc_base.isdir():
+			inc_select = selection.Select(inc_base).set_iter()
+			for inc in inc_select: yield inc
+
+	def get_summary_triples(mirror_total, time_dict):
+		"""Return list of triples (time, size, cumulative size)"""
+		triples = []
+
+		cur_mir_base = Globals.rbdir.append('current_mirror')
+		mirror_time = restore.get_inclist(cur_mir_base)[0].getinctime()
+		triples.append((mirror_time, mirror_total, mirror_total))
+
+		inc_times = time_dict.keys()
+		inc_times.sort()
+		inc_times.reverse()
+		cumulative_size = mirror_total
+		for inc_time in inc_times:
+			size = time_dict[inc_time]
+			cumulative_size += size
+			triples.append((inc_time, size, cumulative_size))
+		return triples
+
+	def triple_to_line(triple):
+		"""Convert triple to display string"""
+		time, size, cum_size = triple
+		return "%24s   %13s   %15s" % \
+			   (Time.timetopretty(time),
+				stat_obj.get_byte_summary_string(size),
+				stat_obj.get_byte_summary_string(cum_size))
+
+	mirror_total = get_total(get_mirror_select())
+	time_dict = get_time_dict(get_inc_select())
+	triples = get_summary_triples(mirror_total, time_dict)
+
+	l = ['%12s %9s  %15s   %20s' % ('Time', '', 'Size', 'Cumulative size'),
+		 '-' * 77,
+		 triple_to_line(triples[0]) + '   (current mirror)']
+	for triple in triples[1:]: l.append(triple_to_line(triple))
+	return '\n'.join(l)
+
+		

trunk/rdiff-backup/testing/finaltest.py

@@ -296,6 +296,62 @@ class Final(PathSetter):
 		assert wc_output.split() == ["0", "0", "0"], wc_output
 
 
+class FinalMisc(PathSetter):
+	"""Test miscellaneous operations like list-increments, etc.
+
+	Many of these just run and make sure there were no errors; they
+	don't verify the output.
+
+	"""
+	def testListIncrementsLocal(self):
+		"""Test --list-increments switch.  Assumes restoretest3 valid rd dir"""
+		self.set_connections(None, None, None, None)
+		self.exec_rb_extra_args(None, "--list-increments",
+								"testfiles/restoretest3")
+
+	def testListIncrementsRemote(self):
+		"""Test --list-increments mode remotely.  Uses restoretest3"""
+		self.set_connections('test1', '../', None, None)
+		self.exec_rb_extra_args(None, "--list-increments",
+								"testfiles/restoretest3")
+
+	def testListChangeSinceLocal(self):
+		"""Test --list-changed-since mode locally.  Uses restoretest3"""
+		self.set_connections(None, None, None, None)
+		self.exec_rb_extra_args(None, '--list-changed-since 10000',
+								'testfiles/restoretest3')
+
+	def testListChangeSinceRemote(self):
+		"""Test --list-changed-since mode remotely.  Uses restoretest3"""
+		self.set_connections('test1', '../', None, None)
+		self.exec_rb_extra_args(None, '--list-changed-since 10000',
+								'testfiles/restoretest3')
+
+	def testListAtTimeLocal(self):
+		"""Test --list-at-time mode locally.  Uses restoretest3"""
+		self.set_connections(None, None, None, None)
+		self.exec_rb_extra_args(None, '--list-at-time 20000',
+								'testfiles/restoretest3')
+		
+	def testListAtTimeRemote(self):
+		"""Test --list-at-time mode locally.  Uses restoretest3"""
+		self.set_connections('test1', '../', None, None)
+		self.exec_rb_extra_args(None, '--list-at-time 20000',
+								'testfiles/restoretest3')
+
+	def testListIncrementSizesLocal(self):
+		"""Test --list-increment-sizes switch.  Uses restoretest3"""
+		self.set_connections(None, None, None, None)
+		self.exec_rb_extra_args(None, "--list-increment-sizes",
+								"testfiles/restoretest3")
+
+	def testListIncrementsRemote(self):
+		"""Test --list-increment-sizes mode remotely.  Uses restoretest3"""
+		self.set_connections('test1', '../', None, None)
+		self.exec_rb_extra_args(None, "--list-increment-sizes",
+								"testfiles/restoretest3")
+
+
 class FinalSelection(PathSetter):
 	"""Test selection options"""
 	def run(self, cmd):