Added hardlink support, refactored some test cases

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@7 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

trunk/rdiff-backup/TODO

@@ -6,3 +6,11 @@ hardlinks
 
 Don't produce stack trace which looks like crash/include file name in
 logging stats
+
+Michael S. Muegel suggestion: "--char-translate source-char
+replacement-string" for use between windows/unix conversions, e.g. ':'
+to _colon_.  Also distinguish new vs changed update in lvl 5 logging.
+
+CB:  Log the filenames skipped (not just excluded) for various reasons
+
+

trunk/rdiff-backup/rdiff-backup.1

@@ -51,6 +51,11 @@ permissions and mtimes afterwards.
 This option controls every how many seconds rdiff-backup checkpoints
 its current status.  The default is 20.
 .TP
+.BI "--current-time " seconds
+This option is useful mainly for testing.  If set, rdiff-backup will
+it for the current time instead of consulting the clock.  The argument
+is the number of seconds since the epoch.
+.TP
 .BI "--exclude " regexp
 Exclude files matching regexp.  This argument can be used multiple times.
 .TP
@@ -67,6 +72,13 @@ automatically excluded.
 Authorize overwriting of a destination directory.  rdiff-backup will
 generally tell you if it needs this.
 .TP
+.BI --hard-links
+Preserve hard links from source to mirror directories.  No increment
+files will themselves be hard linked, but a hard link database will be
+written so that hard links from any dataset will be recreated if
+originally present.  If many hard linked files are present, this
+option can drastically increase memory usage.
+.TP
 .B "-l, --list-increments"
 List the number and date of partial incremental backups contained in
 the specified destination directory.

trunk/rdiff-backup/rdiff_backup/connection.py

@@ -6,11 +6,9 @@ import types, os, tempfile, cPickle, shutil, traceback
 # connection - Code that deals with remote execution
 #
 
-class ConnectionError(Exception):
-	pass
+class ConnectionError(Exception): pass
 
-class ConnectionQuit(Exception):
-	pass
+class ConnectionQuit(Exception): pass
 
 
 class Connection:
@@ -433,6 +431,10 @@ class VirtualFile:
 		return cls.vfiles[id].read(length)
 	readfromid = classmethod(readfromid)
 
+	def readlinefromid(cls, id):
+		return cls.vfiles[id].readline()
+	readlinefromid = classmethod(readlinefromid)
+
 	def writetoid(cls, id, buffer):
 		return cls.vfiles[id].write(buffer)
 	writetoid = classmethod(writetoid)
@@ -460,6 +462,9 @@ class VirtualFile:
 	def read(self, length = -1):
 		return self.connection.VirtualFile.readfromid(self.id, length)
 
+	def readline(self):
+		return self.connection.VirtualFile.readlinefromid(self.id)
+
 	def write(self, buf):
 		return self.connection.VirtualFile.writetoid(self.id, buf)
 

trunk/rdiff-backup/rdiff_backup/highlevel.py

 from __future__ import generators
-execfile("filelist.py")
+execfile("manage.py")
 
 #######################################################################
 #
@@ -61,12 +61,19 @@ class HighLevel:
 		dest_rpath.setdata()
 		inc_rpath.setdata()
 
-	def Restore(rest_time, mirror_base, baseinc_tup, target_base):
+	def Restore(rest_time, mirror_base, rel_index, baseinc_tup, target_base):
 		"""Like Restore.RestoreRecursive but check arguments"""
+		if (Globals.preserve_hardlinks != 0 and
+			Hardlink.retrieve_final(rest_time)):
+			Log("Hard link information found, attempting to preserve "
+				"hard links.", 4)
+			SetConnections.UpdateGlobal('preserve_hardlinks', 1)
+		else: SetConnections.UpdateGlobal('preserve_hardlinks', None)
+
 		if not isinstance(target_base, DSRPath):
 			target_base = DSRPath(target_base.conn, target_base.base,
 								  target_base.index, target_base.data)
-		Restore.RestoreRecursive(rest_time, mirror_base,
+		Restore.RestoreRecursive(rest_time, mirror_base, rel_index,
 								 baseinc_tup, target_base)
 
 MakeStatic(HighLevel)
@@ -154,27 +161,38 @@ class HLDestinationStruct:
 
 		"""
 		collated = RORPIter.CollateIterators(src_init_iter, dest_init_iter)
+		def compare(src_rorp, dest_dsrp):
+			"""Return dest_dsrp if they are different, None if the same"""
+			if not dest_dsrp:
+				dest_dsrp = DSRPath(baserp.conn, baserp.base, src_rorp.index)
+				if dest_dsrp.lstat():
+					Log("Warning: Found unexpected destination file %s, "
+						"not processing it." % dest_dsrp.path, 2)
+					return None
+			elif (src_rorp and src_rorp == dest_dsrp and
+				  (not Globals.preserve_hardlinks or
+				   Hardlink.rorp_eq(src_rorp, dest_dsrp))):
+				return None
+			if src_rorp and src_rorp.isreg() and Hardlink.islinked(src_rorp):
+				dest_dsrp.flaglinked()
+			return dest_dsrp
+
 		def generate_dissimilar():
 			counter = 0
 			for src_rorp, dest_dsrp in collated:
-				if not dest_dsrp:
-					dsrp = DSRPath(baserp.conn, baserp.base, src_rorp.index)
-					if dsrp.lstat():
-						Log("Warning: Found unexpected destination file %s."
-							% dsrp.path, 2)
-						if DestructiveStepping.isexcluded(dsrp, None): continue
+				if Globals.preserve_hardlinks:
+					if src_rorp: Hardlink.add_rorp(src_rorp, 1)
+					if dest_dsrp: Hardlink.add_rorp(dest_dsrp, None)
+				dsrp = compare(src_rorp, dest_dsrp)
+				if dsrp:
 					counter = 0
 					yield dsrp
-				elif not src_rorp or not src_rorp == dest_dsrp:
+				elif counter == 20:
+					placeholder = RORPath(src_rorp.index)
+					placeholder.make_placeholder()
 					counter = 0
-					yield dest_dsrp
-				else: # source and destinition both exist and are same
-					if counter == 20:
-						placeholder = RORPath(src_rorp.index)
-						placeholder.make_placeholder()
-						counter = 0
-						yield placeholder
-					else: counter += 1
+					yield placeholder
+				else: counter += 1
 		return generate_dissimilar()
 
 	def get_sigs(cls, baserp, src_init_iter):
@@ -225,6 +243,8 @@ class HLDestinationStruct:
 				if checkpoint: SaveState.checkpoint_mirror(finalizer, dsrp)
 		except: cls.handle_last_error(dsrp, finalizer)
 		finalizer.getresult()
+		if Globals.preserve_hardlinks and Globals.rbdir:
+			Hardlink.final_writedata()
 		if checkpoint: SaveState.checkpoint_remove()
 
 	def patch_increment_and_finalize(cls, dest_rpath, diffs, inc_rpath):
@@ -258,6 +278,7 @@ class HLDestinationStruct:
 		except: cls.handle_last_error(dsrp, finalizer, ITR)
 		ITR.getresult()
 		finalizer.getresult()
+		if Globals.preserve_hardlinks: Hardlink.final_writedata()
 		SaveState.checkpoint_remove()
 
 	def check_skip_error(cls, thunk):
@@ -282,6 +303,8 @@ class HLDestinationStruct:
 		Log.exception(1)
 		if ITR: SaveState.checkpoint_inc_backup(ITR, finalizer, dsrp, 1)
 		else: SaveState.checkpoint_mirror(finalizer, dsrp, 1)
+		if Globals.preserve_hardlinks:
+			Hardlink.final_checkpoint(Globals.rbdir)
 		SaveState.touch_last_file_definitive()
 		raise
 

trunk/rdiff-backup/rdiff_backup/increment.py

@@ -141,11 +141,14 @@ class Inc:
 
 			"""
 			if diff_rorp:
-				if dsrp.isreg() and diff_rorp.isreg():
+				if diff_rorp.isreg() and (dsrp.isreg() or
+										  diff_rorp.isflaglinked()):
 					tf = TempFileManager.new(dsrp)
 					def init_thunk():
-						Rdiff.patch_with_attribs_action(dsrp, diff_rorp,
-														tf).execute()
+						if diff_rorp.isflaglinked():
+							Hardlink.link_rp(diff_rorp, tf, dsrp)
+						else: Rdiff.patch_with_attribs_action(dsrp, diff_rorp,
+															  tf).execute()
 						Inc.Increment_action(tf, dsrp, incpref).execute()
 					Robust.make_tf_robustaction(init_thunk, (tf,),
 												(dsrp,)).execute()

trunk/rdiff-backup/rdiff_backup/log.py

@@ -40,6 +40,7 @@ class Logger:
 		write commands off to it.
 
 		"""
+		assert not self.log_file_open
 		for conn in Globals.connections:
 			conn.Log.open_logfile_allconn(rpath.conn)
 		rpath.conn.Log.open_logfile_local(rpath)
@@ -71,6 +72,7 @@ class Logger:
 		"""Run by logging connection - close logfile"""
 		assert self.log_file_conn is Globals.local_connection
 		assert not self.logfp.close()
+		self.log_file_local = None
 
 	def format(self, message, verbosity):
 		"""Format the message, possibly adding date information"""

trunk/rdiff-backup/rdiff_backup/restore.py

@@ -10,11 +10,12 @@ import tempfile
 class RestoreError(Exception): pass
 
 class Restore:
-	def RestoreFile(rest_time, rpbase, inclist, rptarget):
+	def RestoreFile(rest_time, rpbase, mirror_rel_index, inclist, rptarget):
 		"""Non-recursive restore function
 
 		rest_time is the time in seconds to restore to,
 		rpbase is the base name of the file being restored,
+		mirror_rel_index is the same as in RestoreRecursive,
 		inclist is a list of rpaths containing all the relevant increments,
 		and rptarget is the rpath that will be written with the restored file.
 
@@ -25,6 +26,12 @@ class Restore:
 		Log("Restoring %s with increments %s to %s" %
 			(rpbase and rpbase.path,
 			 Restore.inclist2str(inclist), rptarget.path), 5)
+
+		if (Globals.preserve_hardlinks and
+			Hardlink.restore_link(mirror_rel_index, rptarget)):
+			RPath.copy_attribs(inclist and inclist[-1] or rpbase, rptarget)
+			return
+
 		if not inclist or inclist[0].getinctype() == "diff":
 			assert rpbase and rpbase.lstat(), \
 				   "No base to go with incs %s" % Restore.inclist2str(inclist)
@@ -73,14 +80,23 @@ class Restore:
 		else: raise RestoreError("Unknown inctype %s" % inctype)
 		RPath.copy_attribs(inc, target)
 
-	def RestoreRecursive(rest_time, mirror_base, baseinc_tup, target_base):
+	def RestoreRecursive(rest_time, mirror_base, mirror_rel_index,
+						 baseinc_tup, target_base):
 		"""Recursive restore function.
 
 		rest_time is the time in seconds to restore to;
+
 		mirror_base is an rpath of the mirror directory corresponding
 		to the one to be restored;
+
+		mirror_rel_index is the index of the mirror_base relative to
+		the root of the mirror directory.  (The mirror_base itself
+		always has index (), as its index must match that of
+		target_base.)
+
 		baseinc_tup is the inc tuple (incdir, list of incs) to be
 		restored;
+
 		and target_base in the dsrp of the target directory.
 
 		"""
@@ -99,7 +115,8 @@ class Restore:
 				inclist = inc_tup[1]
 				target = target_base.new_index(inc_tup.index)
 			DestructiveStepping.initialize(target, None)
-			Restore.RestoreFile(rest_time, mirror, inclist, target)
+			Restore.RestoreFile(rest_time, mirror, mirror_rel_index,
+								inclist, target)
 			target_finalizer(target)
 			if mirror: mirror_finalizer(mirror)
 		target_finalizer.getresult()

trunk/rdiff-backup/rdiff_backup/robust.py

 import tempfile
-execfile("rpath.py")
+execfile("hardlink.py")
 
 #######################################################################
 #
@@ -258,6 +258,16 @@ class TempFile(RPath):
 				rp_dest.chmod(self.getperms())
 				self.chmod(0700)
 		RPathStatic.rename(self, rp_dest)
+
+		# Sometimes this just seems to fail silently, as in one
+		# hardlinked twin is moved over the other.  So check to make
+		# sure below.
+		self.setdata()
+		if self.lstat():
+			rp_dest.delete()
+			RPathStatic.rename(self, rp_dest)
+			self.setdata()
+			if self.lstat(): raise OSError("Cannot rename tmp file correctly")
 		TempFileManager.remove_listing(self)
 
 	def delete(self):
@@ -283,7 +293,8 @@ class SaveState:
 			return Globals.backup_writer.SaveState.init_filenames(incrementing)
 
 		assert Globals.local_connection is Globals.rbdir.conn, \
-			   Globals.rbdir.conn
+			   (Globals.rbdir.conn, Globals.backup_writer)
+
 		if incrementing: cls._last_file_sym = Globals.rbdir.append(
 			"last-file-incremented.%s.snapshot" % Time.curtimestr)
 		else: cls._last_file_sym = Globals.rbdir.append(
@@ -362,6 +373,7 @@ class SaveState:
 	def checkpoint_remove(cls):
 		"""Remove all checkpointing data after successful operation"""
 		for rp in Resume.get_relevant_rps(): rp.delete()
+		if Globals.preserve_hardlinks: Hardlink.remove_all_checkpoints()
 
 MakeClass(SaveState)
 
@@ -506,6 +518,11 @@ class Resume:
 				Log("Resuming aborted backup dated %s" %
 					Time.timetopretty(si.time), 2)
 				Time.setcurtime(si.time)
+				if Globals.preserve_hardlinks:
+					if (not si.last_definitive or not
+						Hardlink.retrieve_checkpoint(Globals.rbdir, si.time)):
+						Log("Hardlink information not successfully "
+							"recovered.", 2)
 				return si
 			else:
 				Log("Last backup dated %s was aborted, but we aren't "

trunk/rdiff-backup/rdiff_backup/rorpiter.py

@@ -61,7 +61,9 @@ class RORPIter:
 			if rp.isplaceholder(): yield rp
 			else:
 				rorp = rp.getRORPath()
-				if rp.isreg(): rorp.setfile(Rdiff.get_signature(rp))
+				if rp.isreg():
+					if rp.isflaglinked(): rorp.flaglinked()
+					else: rorp.setfile(Rdiff.get_signature(rp))
 				yield rorp
 
 	def GetSignatureIter(base_rp):
@@ -172,7 +174,12 @@ class RORPIter:
 
 	def diffonce(sig_rorp, new_rp):
 		"""Return one diff rorp, based from signature rorp and orig rp"""
-		if sig_rorp and sig_rorp.isreg() and new_rp and new_rp.isreg():
+		if sig_rorp and Globals.preserve_hardlinks and sig_rorp.isflaglinked():
+			if new_rp: diff_rorp = new_rp.getRORPath()
+			else: diff_rorp = RORPath(sig_rorp.index)
+			diff_rorp.flaglinked()
+			return diff_rorp
+		elif sig_rorp and sig_rorp.isreg() and new_rp and new_rp.isreg():
 			diff_rorp = new_rp.getRORPath()
 			diff_rorp.setfile(Rdiff.get_delta_sigfileobj(sig_rorp.open("rb"),
 														 new_rp))
@@ -201,7 +208,12 @@ class RORPIter:
 		if not diff_rorp.lstat():
 			return RobustAction(lambda: None, basisrp.delete, lambda e: None)
 
-		if basisrp and basisrp.isreg() and diff_rorp.isreg():
+		if Globals.preserve_hardlinks and diff_rorp.isflaglinked():
+			if not basisrp: basisrp = base_rp.new_index(diff_rorp.index)
+			return RobustAction(lambda: None,
+								lambda: Hardlink.link_rp(diff_rorp, basisrp),
+								lambda e: None)
+		elif basisrp and basisrp.isreg() and diff_rorp.isreg():
 			assert diff_rorp.get_attached_filetype() == 'diff'
 			return Rdiff.patch_with_attribs_action(basisrp, diff_rorp)
 		else: # Diff contains whole file, just copy it over

trunk/rdiff-backup/rdiff_backup/rpath.py

@@ -179,23 +179,6 @@ class RPathStatic:
 		try: return tuple(os.lstat(filename))
 		except os.error: return None
 
-	def cmp_recursive(rp1, rp2):
-		"""True if rp1 and rp2 are at the base of same directories
-
-		Includes only attributes, no file data.  This function may not
-		be used in rdiff-backup but it comes in handy in the unit
-		tests.
-
-		"""
-		rp1.setdata()
-		rp2.setdata()
-		dsiter1, dsiter2 = map(DestructiveStepping.Iterate_with_Finalizer,
-							   [rp1, rp2], [1, None])
-		result = Iter.equal(dsiter1, dsiter2, 1)
-		for i in dsiter1: pass # make sure all files processed anyway
-		for i in dsiter2: pass
-		return result
-
 MakeStatic(RPathStatic)
 
 
@@ -215,15 +198,20 @@ class RORPath(RPathStatic):
 		self.file = None
 
 	def __eq__(self, other):
-		"""Signal two files equivalent"""
-		if not Globals.change_ownership or self.issym() and other.issym():
-			# Don't take file ownership into account when comparing
-			data1, data2 = self.data.copy(), other.data.copy()
-			for d in (data1, data2):
-				for key in ('uid', 'gid'):
-					if d.has_key(key): del d[key]
-			return self.index == other.index and data1 == data2
-		else: return self.index == other.index and self.data == other.data
+		"""True iff the two rorpaths are equivalent"""
+		if self.index != other.index: return None
+
+		for key in self.data.keys(): # compare dicts key by key
+			if ((key == 'uid' or key == 'gid') and
+				(not Globals.change_ownership or self.issym())):
+				# Don't compare gid/uid for symlinks or if not change_ownership
+				pass
+			elif key == 'devloc' or key == 'inode' or key == 'nlink': pass
+			elif (not other.data.has_key(key) or
+				  self.data[key] != other.data[key]): return None
+		return 1
+
+	def __ne__(self, other): return not self.__eq__(other)
 
 	def __str__(self):
 		"""Pretty print file statistics"""
@@ -324,6 +312,18 @@ class RORPath(RPathStatic):
 		"""Return modification time in seconds"""
 		return self.data['mtime']
 	
+	def getinode(self):
+		"""Return inode number of file"""
+		return self.data['inode']
+
+	def getdevloc(self):
+		"""Device number file resides on"""
+		return self.data['devloc']
+
+	def getnumlinks(self):
+		"""Number of places inode is linked to"""
+		return self.data['nlink']
+
 	def readlink(self):
 		"""Wrapper around os.readlink()"""
 		return self.data['linkname']
@@ -352,6 +352,19 @@ class RORPath(RPathStatic):
 		"""Set the type of the attached file"""
 		self.data['filetype'] = type
 
+	def isflaglinked(self):
+		"""True if rorp is a signature/diff for a hardlink file
+
+		This indicates that a file's data need not be transferred
+		because it is hardlinked on the remote side.
+
+		"""
+		return self.data.has_key('linked')
+
+	def flaglinked(self):
+		"""Signal that rorp is a signature/diff for a hardlink file"""
+		self.data['linked'] = 1
+
 	def open(self, mode):
 		"""Return file type object if any was given using self.setfile"""
 		if mode != "rb": raise RPathException("Bad mode %s" % mode)
@@ -447,6 +460,9 @@ class RPath(RORPath):
 		data['perms'] = stat.S_IMODE(mode)
 		data['uid'] = statblock[stat.ST_UID]
 		data['gid'] = statblock[stat.ST_GID]
+		data['inode'] = statblock[stat.ST_INO]
+		data['devloc'] = statblock[stat.ST_DEV]
+		data['nlink'] = statblock[stat.ST_NLINK]
 
 		if not (type == 'sym' or type == 'dev'):
 			# mtimes on symlinks and dev files don't work consistently
@@ -522,6 +538,11 @@ class RPath(RORPath):
 		self.setdata()
 		assert self.issym()
 
+	def hardlink(self, linkpath):
+		"""Make self into a hardlink joined to linkpath"""
+		self.conn.os.link(linkpath, self.path)
+		self.setdata()
+
 	def mkfifo(self):
 		"""Make a fifo at self.path"""
 		self.conn.os.mkfifo(self.path)

trunk/rdiff-backup/src/Make

@@ -21,11 +21,10 @@ def mystrip(filename):
 
 
 files = ["globals.py", "static.py", "lazy.py", "log.py", "ttime.py",
-		 "iterfile.py", "rlist.py", "rdiff.py", "connection.py",
-		 "rpath.py", "robust.py", "rorpiter.py",
+		 "iterfile.py", "rdiff.py", "connection.py", "rpath.py",
+		 "hardlink.py", "robust.py", "rorpiter.py",
 		 "destructive_stepping.py", "increment.py", "restore.py",
-		 "manage.py", "filelist.py", "highlevel.py",
-		 "setconnections.py", "main.py"]
+		 "manage.py", "highlevel.py", "setconnections.py", "main.py"]
 
 os.system("cp header.py rdiff-backup")
 

trunk/rdiff-backup/src/connection.py

@@ -6,11 +6,9 @@ import types, os, tempfile, cPickle, shutil, traceback
 # connection - Code that deals with remote execution
 #
 
-class ConnectionError(Exception):
-	pass
+class ConnectionError(Exception): pass
 
-class ConnectionQuit(Exception):
-	pass
+class ConnectionQuit(Exception): pass
 
 
 class Connection:
@@ -433,6 +431,10 @@ class VirtualFile:
 		return cls.vfiles[id].read(length)
 	readfromid = classmethod(readfromid)
 
+	def readlinefromid(cls, id):
+		return cls.vfiles[id].readline()
+	readlinefromid = classmethod(readlinefromid)
+
 	def writetoid(cls, id, buffer):
 		return cls.vfiles[id].write(buffer)
 	writetoid = classmethod(writetoid)
@@ -460,6 +462,9 @@ class VirtualFile:
 	def read(self, length = -1):
 		return self.connection.VirtualFile.readfromid(self.id, length)
 
+	def readline(self):
+		return self.connection.VirtualFile.readlinefromid(self.id)
+
 	def write(self, buf):
 		return self.connection.VirtualFile.writetoid(self.id, buf)
 

trunk/rdiff-backup/src/globals.py

@@ -10,6 +10,10 @@ class Globals:
 	# The current version of rdiff-backup
 	version = "0.6.0"
 	
+	# If this is set, use this value in seconds as the current time
+	# instead of reading it from the clock.
+	current_time = None
+
 	# This determines how many bytes to read at a time when copying
 	blocksize = 32768
 
@@ -121,6 +125,13 @@ class Globals:
 	# under MS windows NT.
 	time_separator = ":"
 
+	# If true, then hardlinks will be preserved to mirror and recorded
+	# in the increments directory.  There is also a difference here
+	# between None and 0.  When restoring, None or 1 means to preserve
+	# hardlinks iff can find a hardlink dictionary.  0 means ignore
+	# hardlink information regardless.
+	preserve_hardlinks = 1
+
 	def get(cls, name):
 		"""Return the value of something in this class"""
 		return cls.__dict__[name]

trunk/rdiff-backup/src/highlevel.py

 from __future__ import generators
-execfile("filelist.py")
+execfile("manage.py")
 
 #######################################################################
 #
@@ -61,12 +61,19 @@ class HighLevel:
 		dest_rpath.setdata()
 		inc_rpath.setdata()
 
-	def Restore(rest_time, mirror_base, baseinc_tup, target_base):
+	def Restore(rest_time, mirror_base, rel_index, baseinc_tup, target_base):
 		"""Like Restore.RestoreRecursive but check arguments"""
+		if (Globals.preserve_hardlinks != 0 and
+			Hardlink.retrieve_final(rest_time)):
+			Log("Hard link information found, attempting to preserve "
+				"hard links.", 4)
+			SetConnections.UpdateGlobal('preserve_hardlinks', 1)
+		else: SetConnections.UpdateGlobal('preserve_hardlinks', None)
+
 		if not isinstance(target_base, DSRPath):
 			target_base = DSRPath(target_base.conn, target_base.base,
 								  target_base.index, target_base.data)
-		Restore.RestoreRecursive(rest_time, mirror_base,
+		Restore.RestoreRecursive(rest_time, mirror_base, rel_index,
 								 baseinc_tup, target_base)
 
 MakeStatic(HighLevel)
@@ -154,27 +161,38 @@ class HLDestinationStruct:
 
 		"""
 		collated = RORPIter.CollateIterators(src_init_iter, dest_init_iter)
+		def compare(src_rorp, dest_dsrp):
+			"""Return dest_dsrp if they are different, None if the same"""
+			if not dest_dsrp:
+				dest_dsrp = DSRPath(baserp.conn, baserp.base, src_rorp.index)
+				if dest_dsrp.lstat():
+					Log("Warning: Found unexpected destination file %s, "
+						"not processing it." % dest_dsrp.path, 2)
+					return None
+			elif (src_rorp and src_rorp == dest_dsrp and
+				  (not Globals.preserve_hardlinks or
+				   Hardlink.rorp_eq(src_rorp, dest_dsrp))):
+				return None
+			if src_rorp and src_rorp.isreg() and Hardlink.islinked(src_rorp):
+				dest_dsrp.flaglinked()
+			return dest_dsrp
+
 		def generate_dissimilar():
 			counter = 0
 			for src_rorp, dest_dsrp in collated:
-				if not dest_dsrp:
-					dsrp = DSRPath(baserp.conn, baserp.base, src_rorp.index)
-					if dsrp.lstat():
-						Log("Warning: Found unexpected destination file %s."
-							% dsrp.path, 2)
-						if DestructiveStepping.isexcluded(dsrp, None): continue
+				if Globals.preserve_hardlinks:
+					if src_rorp: Hardlink.add_rorp(src_rorp, 1)
+					if dest_dsrp: Hardlink.add_rorp(dest_dsrp, None)
+				dsrp = compare(src_rorp, dest_dsrp)
+				if dsrp:
 					counter = 0
 					yield dsrp
-				elif not src_rorp or not src_rorp == dest_dsrp:
+				elif counter == 20:
+					placeholder = RORPath(src_rorp.index)
+					placeholder.make_placeholder()
 					counter = 0
-					yield dest_dsrp
-				else: # source and destinition both exist and are same
-					if counter == 20:
-						placeholder = RORPath(src_rorp.index)
-						placeholder.make_placeholder()
-						counter = 0
-						yield placeholder
-					else: counter += 1
+					yield placeholder
+				else: counter += 1
 		return generate_dissimilar()
 
 	def get_sigs(cls, baserp, src_init_iter):
@@ -225,6 +243,8 @@ class HLDestinationStruct:
 				if checkpoint: SaveState.checkpoint_mirror(finalizer, dsrp)
 		except: cls.handle_last_error(dsrp, finalizer)
 		finalizer.getresult()
+		if Globals.preserve_hardlinks and Globals.rbdir:
+			Hardlink.final_writedata()
 		if checkpoint: SaveState.checkpoint_remove()
 
 	def patch_increment_and_finalize(cls, dest_rpath, diffs, inc_rpath):
@@ -258,6 +278,7 @@ class HLDestinationStruct:
 		except: cls.handle_last_error(dsrp, finalizer, ITR)
 		ITR.getresult()
 		finalizer.getresult()
+		if Globals.preserve_hardlinks: Hardlink.final_writedata()
 		SaveState.checkpoint_remove()
 
 	def check_skip_error(cls, thunk):
@@ -282,6 +303,8 @@ class HLDestinationStruct:
 		Log.exception(1)
 		if ITR: SaveState.checkpoint_inc_backup(ITR, finalizer, dsrp, 1)
 		else: SaveState.checkpoint_mirror(finalizer, dsrp, 1)
+		if Globals.preserve_hardlinks:
+			Hardlink.final_checkpoint(Globals.rbdir)
 		SaveState.touch_last_file_definitive()
 		raise
 

trunk/rdiff-backup/src/increment.py

@@ -141,11 +141,14 @@ class Inc:
 
 			"""
 			if diff_rorp:
-				if dsrp.isreg() and diff_rorp.isreg():
+				if diff_rorp.isreg() and (dsrp.isreg() or
+										  diff_rorp.isflaglinked()):
 					tf = TempFileManager.new(dsrp)
 					def init_thunk():
-						Rdiff.patch_with_attribs_action(dsrp, diff_rorp,
-														tf).execute()
+						if diff_rorp.isflaglinked():
+							Hardlink.link_rp(diff_rorp, tf, dsrp)
+						else: Rdiff.patch_with_attribs_action(dsrp, diff_rorp,
+															  tf).execute()
 						Inc.Increment_action(tf, dsrp, incpref).execute()
 					Robust.make_tf_robustaction(init_thunk, (tf,),
 												(dsrp,)).execute()

trunk/rdiff-backup/src/log.py

@@ -40,6 +40,7 @@ class Logger:
 		write commands off to it.
 
 		"""
+		assert not self.log_file_open
 		for conn in Globals.connections:
 			conn.Log.open_logfile_allconn(rpath.conn)
 		rpath.conn.Log.open_logfile_local(rpath)
@@ -71,6 +72,7 @@ class Logger:
 		"""Run by logging connection - close logfile"""
 		assert self.log_file_conn is Globals.local_connection
 		assert not self.logfp.close()
+		self.log_file_local = None
 
 	def format(self, message, verbosity):
 		"""Format the message, possibly adding date information"""

trunk/rdiff-backup/src/main.py

 #!/usr/bin/python
 
-execfile("highlevel.py")
+execfile("setconnections.py")
 import getopt, sys, re
 
 #######################################################################
@@ -27,7 +27,7 @@ class Main:
 			  "include-from-stdin", "terminal-verbosity=",
 			  "exclude-device-files", "resume", "no-resume",
 			  "resume-window=", "windows-time-format",
-			  "checkpoint-interval="])
+			  "checkpoint-interval=", "hard-links", "current-time="])
 		except getopt.error:
 			self.commandline_error("Error parsing commandline options")
 
@@ -37,12 +37,15 @@ class Main:
 				Globals.set('change_source_perms', 1)
 			elif opt == "--checkpoint-interval":
 				Globals.set_integer('checkpoint_interval', arg)
+			elif opt == "--current-time":
+				Globals.set_integer('current_time', arg)
 			elif opt == "--exclude": self.exclude_regstrs.append(arg)
 			elif opt == "--exclude-device-files":
 				Globals.set('exclude_device_files', 1)
 			elif opt == "--exclude-mirror":
 				self.exclude_mirror_regstrs.append(arg)
 			elif opt == "--force": self.force = 1
+			elif opt == "--hard-links": Globals.set('preserve_hardlinks', 1)
 			elif opt == "--include-from-stdin": Globals.include_from_stdin = 1
 			elif opt == "-l" or opt == "--list-increments":
 				self.action = "list-increments"
@@ -167,7 +170,7 @@ rdiff-backup with the --force option if you want to mirror anyway.""" %
 		"""Backup, possibly incrementally, src_path to dest_path."""
 		SetConnections.BackupInitConnections(rpin.conn, rpout.conn)
 		self.backup_init_dirs(rpin, rpout)
-		Time.setcurtime()
+		Time.setcurtime(Globals.current_time)
 		RSI = Resume.ResumeCheck()
 		if self.prevtime:
 			Time.setprevtime(self.prevtime)
@@ -213,6 +216,7 @@ rdiff-backup with the --force option.""" % rpout.path)
 		if not self.datadir.lstat(): self.datadir.mkdir()
 		Globals.add_regexp(self.datadir.path, 1)
 		Globals.add_regexp(rpin.append("rdiff-backup-data").path, None)
+
 		if Log.verbosity > 0:
 			Log.open_logfile(self.datadir.append("backup.log"))
 		self.backup_warn_if_infinite_regress(rpin, rpout)
@@ -271,10 +275,10 @@ went wrong during your last backup?  Using """ + mirrorrps[-1].path, 2)
 		Log("Starting Restore", 5)
 		rpin, rpout = self.restore_check_paths(src_rp, dest_rp)
 		inc_tup = self.restore_get_inctup(rpin)
-		mirror_base = self.restore_get_mirror(rpin)
+		mirror_base, mirror_rel_index = self.restore_get_mirror(rpin)
 		rtime = Time.stringtotime(rpin.getinctime())
 		Log.open_logfile(self.datadir.append("restore.log"))
-		HighLevel.Restore(rtime, mirror_base, inc_tup, rpout)
+		HighLevel.Restore(rtime, mirror_base, mirror_rel_index, inc_tup, rpout)
 
 	def restore_check_paths(self, rpin, rpout):
 		"""Check paths and return pair of corresponding rps"""
@@ -306,7 +310,7 @@ Try restoring from an increment file (the filenames look like
 		return IndexedTuple((), (incbase, inclist))
 
 	def restore_get_mirror(self, rpin):
-		"""Return mirror file and set the data dir
+		"""Return (mirror file, relative index) and set the data dir
 
 		The idea here is to keep backing up on the path until we find
 		something named "rdiff-backup-data".  Then use that as a
@@ -314,6 +318,9 @@ Try restoring from an increment file (the filenames look like
 		increment file is pointed to in a funny way, using symlinks or
 		somesuch.
 
+		The mirror file will have index (), so also return the index
+		relative to the rootrp.
+
 		"""
 		pathcomps = os.path.join(rpin.conn.os.getcwd(),
 								 rpin.getincbase().path).split("/")
@@ -323,7 +330,7 @@ Try restoring from an increment file (the filenames look like
 				break
 		else: Log.FatalError("Unable to find rdiff-backup-data dir")
 
-		self.datadir = datadirrp
+		Globals.rbdir = self.datadir = datadirrp
 		Globals.add_regexp(self.datadir.path, 1)
 		rootrp = RPath(rpin.conn, "/".join(pathcomps[:i]))
 		if not rootrp.lstat():
@@ -332,14 +339,12 @@ Try restoring from an increment file (the filenames look like
 		else: Log("Using root mirror %s" % rootrp.path, 6)
 
 		from_datadir = pathcomps[i+1:]
-		if len(from_datadir) == 1: result = rootrp
-		elif len(from_datadir) > 1:
-			result = RPath(rootrp.conn, apply(os.path.join,
-									      [rootrp.path] + from_datadir[1:]))
-		else: raise RestoreError("Problem finding mirror file")
-
-		Log("Using mirror file %s" % result.path, 6)
-		return result
+		if not from_datadir: raise RestoreError("Problem finding mirror file")
+		rel_index = tuple(from_datadir[1:])
+		mirrorrp = RPath(rootrp.conn,
+						 apply(os.path.join, (rootrp.path,) + rel_index))
+		Log("Using mirror file %s" % mirrorrp.path, 6)
+		return (mirrorrp, rel_index)
 
 
 	def ListIncrements(self, rootrp):
@@ -396,6 +401,6 @@ Try finding the increments first using --list-increments.""")
 
 
 
-if __name__ == "__main__":
+if __name__ == "__main__" and not globals().has_key('__no_execute__'):
 	Globals.Main = Main()
 	Globals.Main.Main()

trunk/rdiff-backup/src/rdiff.py

-execfile("rlist.py")
+execfile("iterfile.py")
 import os, popen2
 
 #######################################################################

trunk/rdiff-backup/src/restore.py

@@ -10,11 +10,12 @@ import tempfile
 class RestoreError(Exception): pass
 
 class Restore:
-	def RestoreFile(rest_time, rpbase, inclist, rptarget):
+	def RestoreFile(rest_time, rpbase, mirror_rel_index, inclist, rptarget):
 		"""Non-recursive restore function
 
 		rest_time is the time in seconds to restore to,
 		rpbase is the base name of the file being restored,
+		mirror_rel_index is the same as in RestoreRecursive,
 		inclist is a list of rpaths containing all the relevant increments,
 		and rptarget is the rpath that will be written with the restored file.
 
@@ -25,6 +26,12 @@ class Restore:
 		Log("Restoring %s with increments %s to %s" %
 			(rpbase and rpbase.path,
 			 Restore.inclist2str(inclist), rptarget.path), 5)
+
+		if (Globals.preserve_hardlinks and
+			Hardlink.restore_link(mirror_rel_index, rptarget)):
+			RPath.copy_attribs(inclist and inclist[-1] or rpbase, rptarget)
+			return
+
 		if not inclist or inclist[0].getinctype() == "diff":
 			assert rpbase and rpbase.lstat(), \
 				   "No base to go with incs %s" % Restore.inclist2str(inclist)
@@ -73,14 +80,23 @@ class Restore:
 		else: raise RestoreError("Unknown inctype %s" % inctype)
 		RPath.copy_attribs(inc, target)
 
-	def RestoreRecursive(rest_time, mirror_base, baseinc_tup, target_base):
+	def RestoreRecursive(rest_time, mirror_base, mirror_rel_index,
+						 baseinc_tup, target_base):
 		"""Recursive restore function.
 
 		rest_time is the time in seconds to restore to;
+
 		mirror_base is an rpath of the mirror directory corresponding
 		to the one to be restored;
+
+		mirror_rel_index is the index of the mirror_base relative to
+		the root of the mirror directory.  (The mirror_base itself
+		always has index (), as its index must match that of
+		target_base.)
+
 		baseinc_tup is the inc tuple (incdir, list of incs) to be
 		restored;
+
 		and target_base in the dsrp of the target directory.
 
 		"""
@@ -99,7 +115,8 @@ class Restore:
 				inclist = inc_tup[1]
 				target = target_base.new_index(inc_tup.index)
 			DestructiveStepping.initialize(target, None)
-			Restore.RestoreFile(rest_time, mirror, inclist, target)
+			Restore.RestoreFile(rest_time, mirror, mirror_rel_index,
+								inclist, target)
 			target_finalizer(target)
 			if mirror: mirror_finalizer(mirror)
 		target_finalizer.getresult()

trunk/rdiff-backup/src/robust.py

 import tempfile
-execfile("rpath.py")
+execfile("hardlink.py")
 
 #######################################################################
 #
@@ -258,6 +258,16 @@ class TempFile(RPath):
 				rp_dest.chmod(self.getperms())
 				self.chmod(0700)
 		RPathStatic.rename(self, rp_dest)
+
+		# Sometimes this just seems to fail silently, as in one
+		# hardlinked twin is moved over the other.  So check to make
+		# sure below.
+		self.setdata()
+		if self.lstat():
+			rp_dest.delete()
+			RPathStatic.rename(self, rp_dest)
+			self.setdata()
+			if self.lstat(): raise OSError("Cannot rename tmp file correctly")
 		TempFileManager.remove_listing(self)
 
 	def delete(self):
@@ -283,7 +293,8 @@ class SaveState:
 			return Globals.backup_writer.SaveState.init_filenames(incrementing)
 
 		assert Globals.local_connection is Globals.rbdir.conn, \
-			   Globals.rbdir.conn
+			   (Globals.rbdir.conn, Globals.backup_writer)
+
 		if incrementing: cls._last_file_sym = Globals.rbdir.append(
 			"last-file-incremented.%s.snapshot" % Time.curtimestr)
 		else: cls._last_file_sym = Globals.rbdir.append(
@@ -362,6 +373,7 @@ class SaveState:
 	def checkpoint_remove(cls):
 		"""Remove all checkpointing data after successful operation"""
 		for rp in Resume.get_relevant_rps(): rp.delete()
+		if Globals.preserve_hardlinks: Hardlink.remove_all_checkpoints()
 
 MakeClass(SaveState)
 
@@ -506,6 +518,11 @@ class Resume:
 				Log("Resuming aborted backup dated %s" %
 					Time.timetopretty(si.time), 2)
 				Time.setcurtime(si.time)
+				if Globals.preserve_hardlinks:
+					if (not si.last_definitive or not
+						Hardlink.retrieve_checkpoint(Globals.rbdir, si.time)):
+						Log("Hardlink information not successfully "
+							"recovered.", 2)
 				return si
 			else:
 				Log("Last backup dated %s was aborted, but we aren't "

trunk/rdiff-backup/src/rorpiter.py

@@ -61,7 +61,9 @@ class RORPIter:
 			if rp.isplaceholder(): yield rp
 			else:
 				rorp = rp.getRORPath()
-				if rp.isreg(): rorp.setfile(Rdiff.get_signature(rp))
+				if rp.isreg():
+					if rp.isflaglinked(): rorp.flaglinked()
+					else: rorp.setfile(Rdiff.get_signature(rp))
 				yield rorp
 
 	def GetSignatureIter(base_rp):
@@ -172,7 +174,12 @@ class RORPIter:
 
 	def diffonce(sig_rorp, new_rp):
 		"""Return one diff rorp, based from signature rorp and orig rp"""
-		if sig_rorp and sig_rorp.isreg() and new_rp and new_rp.isreg():
+		if sig_rorp and Globals.preserve_hardlinks and sig_rorp.isflaglinked():
+			if new_rp: diff_rorp = new_rp.getRORPath()
+			else: diff_rorp = RORPath(sig_rorp.index)
+			diff_rorp.flaglinked()
+			return diff_rorp
+		elif sig_rorp and sig_rorp.isreg() and new_rp and new_rp.isreg():
 			diff_rorp = new_rp.getRORPath()
 			diff_rorp.setfile(Rdiff.get_delta_sigfileobj(sig_rorp.open("rb"),
 														 new_rp))
@@ -201,7 +208,12 @@ class RORPIter:
 		if not diff_rorp.lstat():
 			return RobustAction(lambda: None, basisrp.delete, lambda e: None)
 
-		if basisrp and basisrp.isreg() and diff_rorp.isreg():
+		if Globals.preserve_hardlinks and diff_rorp.isflaglinked():
+			if not basisrp: basisrp = base_rp.new_index(diff_rorp.index)
+			return RobustAction(lambda: None,
+								lambda: Hardlink.link_rp(diff_rorp, basisrp),
+								lambda e: None)
+		elif basisrp and basisrp.isreg() and diff_rorp.isreg():
 			assert diff_rorp.get_attached_filetype() == 'diff'
 			return Rdiff.patch_with_attribs_action(basisrp, diff_rorp)
 		else: # Diff contains whole file, just copy it over

trunk/rdiff-backup/src/rpath.py

@@ -179,23 +179,6 @@ class RPathStatic:
 		try: return tuple(os.lstat(filename))
 		except os.error: return None
 
-	def cmp_recursive(rp1, rp2):
-		"""True if rp1 and rp2 are at the base of same directories
-
-		Includes only attributes, no file data.  This function may not
-		be used in rdiff-backup but it comes in handy in the unit
-		tests.
-
-		"""
-		rp1.setdata()
-		rp2.setdata()
-		dsiter1, dsiter2 = map(DestructiveStepping.Iterate_with_Finalizer,
-							   [rp1, rp2], [1, None])
-		result = Iter.equal(dsiter1, dsiter2, 1)
-		for i in dsiter1: pass # make sure all files processed anyway
-		for i in dsiter2: pass
-		return result
-
 MakeStatic(RPathStatic)
 
 
@@ -215,15 +198,20 @@ class RORPath(RPathStatic):
 		self.file = None
 
 	def __eq__(self, other):
-		"""Signal two files equivalent"""
-		if not Globals.change_ownership or self.issym() and other.issym():
-			# Don't take file ownership into account when comparing
-			data1, data2 = self.data.copy(), other.data.copy()
-			for d in (data1, data2):
-				for key in ('uid', 'gid'):
-					if d.has_key(key): del d[key]
-			return self.index == other.index and data1 == data2
-		else: return self.index == other.index and self.data == other.data
+		"""True iff the two rorpaths are equivalent"""
+		if self.index != other.index: return None
+
+		for key in self.data.keys(): # compare dicts key by key
+			if ((key == 'uid' or key == 'gid') and
+				(not Globals.change_ownership or self.issym())):
+				# Don't compare gid/uid for symlinks or if not change_ownership
+				pass
+			elif key == 'devloc' or key == 'inode' or key == 'nlink': pass
+			elif (not other.data.has_key(key) or
+				  self.data[key] != other.data[key]): return None
+		return 1
+
+	def __ne__(self, other): return not self.__eq__(other)
 
 	def __str__(self):
 		"""Pretty print file statistics"""
@@ -324,6 +312,18 @@ class RORPath(RPathStatic):
 		"""Return modification time in seconds"""
 		return self.data['mtime']
 	
+	def getinode(self):
+		"""Return inode number of file"""
+		return self.data['inode']
+
+	def getdevloc(self):
+		"""Device number file resides on"""
+		return self.data['devloc']
+
+	def getnumlinks(self):
+		"""Number of places inode is linked to"""
+		return self.data['nlink']
+
 	def readlink(self):
 		"""Wrapper around os.readlink()"""
 		return self.data['linkname']
@@ -352,6 +352,19 @@ class RORPath(RPathStatic):
 		"""Set the type of the attached file"""
 		self.data['filetype'] = type
 
+	def isflaglinked(self):
+		"""True if rorp is a signature/diff for a hardlink file
+
+		This indicates that a file's data need not be transferred
+		because it is hardlinked on the remote side.
+
+		"""
+		return self.data.has_key('linked')
+
+	def flaglinked(self):
+		"""Signal that rorp is a signature/diff for a hardlink file"""
+		self.data['linked'] = 1
+
 	def open(self, mode):
 		"""Return file type object if any was given using self.setfile"""
 		if mode != "rb": raise RPathException("Bad mode %s" % mode)
@@ -447,6 +460,9 @@ class RPath(RORPath):
 		data['perms'] = stat.S_IMODE(mode)
 		data['uid'] = statblock[stat.ST_UID]
 		data['gid'] = statblock[stat.ST_GID]
+		data['inode'] = statblock[stat.ST_INO]
+		data['devloc'] = statblock[stat.ST_DEV]
+		data['nlink'] = statblock[stat.ST_NLINK]
 
 		if not (type == 'sym' or type == 'dev'):
 			# mtimes on symlinks and dev files don't work consistently
@@ -522,6 +538,11 @@ class RPath(RORPath):
 		self.setdata()
 		assert self.issym()
 
+	def hardlink(self, linkpath):
+		"""Make self into a hardlink joined to linkpath"""
+		self.conn.os.link(linkpath, self.path)
+		self.setdata()
+
 	def mkfifo(self):
 		"""Make a fifo at self.path"""
 		self.conn.os.mkfifo(self.path)

trunk/rdiff-backup/testing/commontest.py

-"""commontest - Some functions and constants common to all test cases"""
+"""commontest - Some functions and constants common to several test cases"""
 import os
 
 SourceDir = "../src"
 AbsCurdir = os.getcwd() # Absolute path name of current directory
 AbsTFdir = AbsCurdir+"/testfiles"
 MiscDir = "../misc"
+__no_execute__ = 1 # Keeps the actual rdiff-backup program from running
 
 def rbexec(src_file):
 	"""Changes to the source directory, execfile src_file, return"""
@@ -17,3 +18,231 @@ def Make():
 	os.chdir(SourceDir)
 	os.system("python ./Make")
 	os.chdir(AbsCurdir)
+
+def rdiff_backup(source_local, dest_local, src_dir, dest_dir,
+				 current_time = None, extra_options = ""):
+	"""Run rdiff-backup with the given options
+
+	source_local and dest_local are boolean values.  If either is
+	false, then rdiff-backup will be run pretending that src_dir and
+	dest_dir, respectively, are remote.  The server process will be
+	run in directories test1 and test2/tmp respectively.
+
+	src_dir and dest_dir are the source and destination
+	(mirror) directories, relative to the testing directory.
+
+	If current time is true, add the --current-time option with the
+	given number of seconds.
+
+	extra_options are just added to the command line.
+
+	"""
+	if not source_local:
+		src_dir = ("cd test1; ../%s/rdiff-backup --server::../%s" %
+				   (SourceDir, src_dir))
+	if not dest_local:
+		dest_dir = ("test2/tmp; ../../%s/rdiff-backup --server::../../%s" %
+					(SourceDir, dest_dir))
+
+	cmdargs = [SourceDir + "/rdiff-backup", extra_options]
+	if not (source_local and dest_local): cmdargs.append("--remote-schema %s")
+
+	if current_time: cmdargs.append("--current-time %s" % current_time)
+
+	os.system(" ".join(cmdargs))	
+
+def InternalBackup(source_local, dest_local, src_dir, dest_dir,
+				   current_time = None):
+	"""Backup src to dest internally
+
+	This is like rdiff_backup but instead of running a separate
+	rdiff-backup script, use the separate *.py files.  This way the
+	script doesn't have to be rebuild constantly, and stacktraces have
+	correct line/file references.
+
+	"""
+	Globals.current_time = current_time
+	#_reset_connections()
+	remote_schema = '%s'
+
+	if not source_local:
+		src_dir = "cd test1; python ../server.py ../%s::../%s" % \
+				  (SourceDir, src_dir)
+	if not dest_local:
+		dest_dir = "cd test2/tmp; python ../../server.py ../../%s::../../%s" \
+				   % (SourceDir, dest_dir)
+
+	rpin, rpout = SetConnections.InitRPs([src_dir, dest_dir], remote_schema)
+	_get_main().Backup(rpin, rpout)
+	_get_main().cleanup()
+
+def InternalMirror(source_local, dest_local, src_dir, dest_dir,
+				   checkpointing = None):
+	"""Mirror src to dest internally, like InternalBackup"""
+	remote_schema = '%s'
+
+	if not source_local:
+		src_dir = "cd test1; python ../server.py ../%s::../%s" % \
+				  (SourceDir, src_dir)
+	if not dest_local:
+		dest_dir = "cd test2/tmp; python ../../server.py ../../%s::../../%s" \
+				   % (SourceDir, dest_dir)
+
+	rpin, rpout = SetConnections.InitRPs([src_dir, dest_dir], remote_schema)
+	if not rpout.lstat(): rpout.mkdir()
+	if checkpointing: # rdiff-backup-data must exist to checkpoint
+		data_dir = rpout.append("rdiff-backup-data")
+		if not data_dir.lstat(): data_dir.mkdir()
+		Globals.add_regexp(data_dir.path, 1)
+		SetConnections.UpdateGlobal('rbdir', data_dir)
+	HighLevel.Mirror(rpin, rpout, checkpointing)
+	_get_main().cleanup()
+
+def InternalRestore(mirror_local, dest_local, mirror_dir, dest_dir, time):
+	"""Restore mirror_dir to dest_dir at given time
+
+	This will automatically find the increments.XXX.dir representing
+	the time specified.  The mirror_dir and dest_dir are relative to
+	the testing directory and will be modified for remote trials.
+
+	"""
+	remote_schema = '%s'
+	#_reset_connections()
+	if not mirror_local:
+		mirror_dir = "cd test1; python ../server.py ../%s::../%s" % \
+					 (SourceDir, mirror_dir)
+	if not dest_local:
+		dest_dir = "cd test2/tmp; python ../../server.py ../../%s::../../%s" \
+				   % (SourceDir, dest_dir)
+
+	mirror_rp, dest_rp = SetConnections.InitRPs([mirror_dir, dest_dir],
+												remote_schema)
+
+	def get_increment_rp(time):
+		"""Return increment rp matching time"""
+		data_rp = mirror_rp.append("rdiff-backup-data")
+		for filename in data_rp.listdir():
+			rp = data_rp.append(filename)
+			if (rp.isincfile() and rp.getincbase_str() == "increments" and
+				Time.stringtotime(rp.getinctime()) == time):
+				return rp
+		assert None, ("No increments.XXX.dir found in directory "
+					  "%s with that time" % data_rp.path)
+
+	_get_main().Restore(get_increment_rp(time), dest_rp)
+	_get_main().cleanup()
+
+def _reset_connections(src_rp, dest_rp):
+	"""Reset some global connection information"""
+	Globals.isbackup_reader = Globals.isbackup_writer = None
+	#Globals.connections = [Globals.local_connection]
+	#Globals.connection_dict = {0: Globals.local_connection}
+	SetConnections.UpdateGlobal('rbdir', None)
+	SetConnections.UpdateGlobal('exclude_regexps', [])
+	SetConnections.UpdateGlobal('exclude_mirror_regexps', [])
+	Globals.add_regexp(dest_rp.append("rdiff-backup-data").path, 1)
+	Globals.add_regexp(src_rp.append("rdiff-backup-data").path, None)
+
+def _get_main():
+	"""Set Globals.Main if it doesn't exist, and return"""
+	try: return Globals.Main
+	except AttributeError:
+		Globals.Main = Main()
+		return Globals.Main
+
+def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1):
+	"""Compare src_rp and dest_rp, which can be directories
+
+	This only compares file attributes, not the actual data.  This
+	will overwrite the hardlink dictionaries if compare_hardlinks is
+	specified.
+
+	"""
+	if compare_hardlinks: reset_hardlink_dicts()
+	src_rp.setdata()
+	dest_rp.setdata()
+
+	Log("Comparing %s and %s, hardlinks %s" % (src_rp.path, dest_rp.path,
+											   compare_hardlinks), 3)
+	dsiter1, dsiter2 = map(DestructiveStepping.Iterate_with_Finalizer,
+						   [src_rp, dest_rp], [1, None])
+
+	def hardlink_equal(src_rorp, dest_rorp):
+		if src_rorp != dest_rorp: return None
+		if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1
+		Log("%s: %s" % (src_rorp.index, Hardlink.get_indicies(src_rorp, 1)), 3)
+		Log("%s: %s" % (dest_rorp.index,
+						Hardlink.get_indicies(dest_rorp, None)), 3)
+		return None
+
+	if compare_hardlinks:
+		dsiter1 = Hardlink.add_rorp_iter(dsiter1, 1)
+		dsiter2 = Hardlink.add_rorp_iter(dsiter2, None)
+		result = Iter.equal(dsiter1, dsiter2, 1, hardlink_equal)
+	else: result = Iter.equal(dsiter1, dsiter2, 1)
+
+	for i in dsiter1: pass # make sure all files processed anyway
+	for i in dsiter2: pass
+	return result
+
+def reset_hardlink_dicts():
+	"""Clear the hardlink dictionaries"""
+	Hardlink._src_inode_indicies = {}
+	Hardlink._src_index_indicies = {}
+	Hardlink._dest_inode_indicies = {}
+	Hardlink._dest_index_indicies = {}
+
+def BackupRestoreSeries(source_local, dest_local, list_of_dirnames,
+						compare_hardlinks = 1,
+						dest_dirname = "testfiles/output",
+						restore_dirname = "testfiles/rest_out"):
+	"""Test backing up/restoring of a series of directories
+
+	The dirnames correspond to a single directory at different times.
+	After each backup, the dest dir will be compared.  After the whole
+	set, each of the earlier directories will be recovered to the
+	restore_dirname and compared.
+
+	"""
+	Globals.set('preserve_hardlinks', compare_hardlinks)
+	time = 10000
+	dest_rp = RPath(Globals.local_connection, dest_dirname)
+	restore_rp = RPath(Globals.local_connection, restore_dirname)
+	
+	os.system(MiscDir + "/myrm " + dest_dirname)
+	for dirname in list_of_dirnames:
+		src_rp = RPath(Globals.local_connection, dirname)
+		reset_hardlink_dicts()
+		_reset_connections(src_rp, dest_rp)
+
+		InternalBackup(source_local, dest_local, dirname, dest_dirname, time)
+		time += 10000
+		_reset_connections(src_rp, dest_rp)
+		assert CompareRecursive(src_rp, dest_rp, compare_hardlinks)
+
+	time = 10000
+	for dirname in list_of_dirnames[:-1]:
+		reset_hardlink_dicts()
+		os.system(MiscDir + "/myrm " + restore_dirname)
+		InternalRestore(dest_local, source_local, dest_dirname,
+						restore_dirname, time)
+		src_rp = RPath(Globals.local_connection, dirname)
+		assert CompareRecursive(src_rp, restore_rp)
+		time += 10000
+
+def MirrorTest(source_local, dest_local, list_of_dirnames,
+			   compare_hardlinks = 1,
+			   dest_dirname = "testfiles/output"):
+	"""Mirror each of list_of_dirnames, and compare after each"""
+	Globals.set('preserve_hardlinks', compare_hardlinks)
+	dest_rp = RPath(Globals.local_connection, dest_dirname)
+
+	os.system(MiscDir + "/myrm " + dest_dirname)
+	for dirname in list_of_dirnames:
+		src_rp = RPath(Globals.local_connection, dirname)
+		reset_hardlink_dicts()
+		_reset_connections(src_rp, dest_rp)
+
+		InternalMirror(source_local, dest_local, dirname, dest_dirname)
+		_reset_connections(src_rp, dest_rp)
+		assert CompareRecursive(src_rp, dest_rp, compare_hardlinks)

trunk/rdiff-backup/testing/finaltest.py

@@ -73,22 +73,22 @@ class PathSetter(unittest.TestCase):
 
 		# Backing up increment1
 		self.exec_rb('testfiles/increment1', 'testfiles/output')
-		assert RPathStatic.cmp_recursive(Local.inc1rp, Local.rpout)
+		assert CompareRecursive(Local.inc1rp, Local.rpout)
 		time.sleep(1)
 
 		# Backing up increment2
 		self.exec_rb('testfiles/increment2', 'testfiles/output')
-		assert RPathStatic.cmp_recursive(Local.inc2rp, Local.rpout)
+		assert CompareRecursive(Local.inc2rp, Local.rpout)
 		time.sleep(1)
 
 		# Backing up increment3
 		self.exec_rb('testfiles/increment3', 'testfiles/output')
-		assert RPathStatic.cmp_recursive(Local.inc3rp, Local.rpout)
+		assert CompareRecursive(Local.inc3rp, Local.rpout)
 		time.sleep(1)
 
 		# Backing up increment4
 		self.exec_rb('testfiles/increment4', 'testfiles/output')
-		assert RPathStatic.cmp_recursive(Local.inc4rp, Local.rpout)
+		assert CompareRecursive(Local.inc4rp, Local.rpout)
 
 		# Getting restore rps
 		inc_paths = self.getinc_paths("increments.",
@@ -97,22 +97,22 @@ class PathSetter(unittest.TestCase):
 
 		# Restoring increment1
 		self.exec_rb(inc_paths[0], 'testfiles/restoretarget1')
-		assert RPathStatic.cmp_recursive(Local.inc1rp, Local.rpout1)
+		assert CompareRecursive(Local.inc1rp, Local.rpout1)
 
 		# Restoring increment2
 		self.exec_rb(inc_paths[1], 'testfiles/restoretarget2')
-		assert RPathStatic.cmp_recursive(Local.inc2rp, Local.rpout2)
+		assert CompareRecursive(Local.inc2rp, Local.rpout2)
 
 		# Restoring increment3
 		self.exec_rb(inc_paths[2], 'testfiles/restoretarget3')
-		assert RPathStatic.cmp_recursive(Local.inc3rp, Local.rpout3)
+		assert CompareRecursive(Local.inc3rp, Local.rpout3)
 
 		# Test restoration of a few random files
 		vft_paths = self.getinc_paths("various_file_types.",
 					     "testfiles/output/rdiff-backup-data/increments")
 		self.exec_rb(vft_paths[1], 'testfiles/vft_out')
 		self.refresh(Local.vft_in, Local.vft_out)
-		assert RPathStatic.cmp_recursive(Local.vft_in, Local.vft_out)
+		assert CompareRecursive(Local.vft_in, Local.vft_out)
 
 		timbar_paths = self.getinc_paths("timbar.pyc.",
 						 "testfiles/output/rdiff-backup-data/increments")

trunk/rdiff-backup/testing/highleveltest.py

 import unittest
 
 execfile("commontest.py")
-rbexec("setconnections.py")
+rbexec("main.py")
 
 
 class RemoteMirrorTest(unittest.TestCase):
@@ -10,66 +10,27 @@ class RemoteMirrorTest(unittest.TestCase):
 		"""Start server"""
 		Log.setverbosity(7)
 		Globals.change_source_perms = 1
-		self.conn = SetConnections.init_connection("./server.py")
-
-		self.inrp = RPath(Globals.local_connection, "testfiles/various_file_types")
-		self.outrp = RPath(self.conn, "testfiles/output")
-		self.rbdir = RPath(self.conn, "testfiles/output/rdiff-backup-data")
-		SetConnections.UpdateGlobal('rbdir', self.rbdir)
-		self.inc1 = RPath(Globals.local_connection, "testfiles/increment1")
-		self.inc2 = RPath(Globals.local_connection, "testfiles/increment2")
-		self.inc3 = RPath(Globals.local_connection, "testfiles/increment3")
-		self.inc4 = RPath(Globals.local_connection, "testfiles/increment4")
-
-		SetConnections.BackupInitConnections(Globals.local_connection,
-											 self.conn)
 		SetConnections.UpdateGlobal('checkpoint_interval', 3)
 
 	def testMirror(self):
 		"""Testing simple mirror"""
-		if self.outrp.lstat(): self.outrp.delete()
-		HighLevel.Mirror(self.inrp, self.outrp, None)
-		self.outrp.setdata()
-		assert RPath.cmp_recursive(self.inrp, self.outrp)
+		MirrorTest(None, None, ["testfiles/increment1"])
 
 	def testMirror2(self):
 		"""Test mirror with larger data set"""
-		if self.outrp.lstat(): self.outrp.delete()
-		for rp in [self.inc1, self.inc2, self.inc3, self.inc4]:
-			rp.setdata()
-			print "----------------- Starting ", rp.path
-			HighLevel.Mirror(rp, self.outrp, None)
-			#if rp is self.inc2: assert 0
-			assert RPath.cmp_recursive(rp, self.outrp)
-			self.outrp.setdata()
+		MirrorTest(1, None, ['testfiles/increment1', 'testfiles/increment2',
+							 'testfiles/increment3', 'testfiles/increment4'])
 
 	def testMirrorWithCheckpointing(self):
 		"""Like testMirror but this time checkpoint"""
-		if self.outrp.lstat(): self.outrp.delete()
-		self.outrp.mkdir()
-		self.rbdir.mkdir()
-		Globals.add_regexp("testfiles/output/rdiff-backup-data", 1)
-		Time.setcurtime()
-		SaveState.init_filenames(None)
-		HighLevel.Mirror(self.inrp, self.outrp, 1)
-		self.outrp.setdata()
-		assert RPath.cmp_recursive(self.inrp, self.outrp)
+		MirrorTest(None, None, ["testfiles/increment1"], 1)
 
 	def testMirrorWithCheckpointing2(self):
 		"""Larger data set"""
-		if self.outrp.lstat(): os.system(MiscDir+"/myrm %s" % self.outrp.path)
-		self.outrp.setdata()
-		self.outrp.mkdir()
-		self.rbdir.mkdir()
-		Globals.add_regexp("testfiles/output/rdiff-backup-data", 1)
-		Time.setcurtime()
-		SaveState.init_filenames(None)
-		for rp in [self.inc1, self.inc2, self.inc3, self.inc4]:
-			print "----------------- Starting ", rp.path
-			HighLevel.Mirror(rp, self.outrp, 1)
-			assert RPath.cmp_recursive(rp, self.outrp)
+		MirrorTest(1, None, ['testfiles/increment1', 'testfiles/increment2',
+							 'testfiles/increment3', 'testfiles/increment4'],
+				   1)
 
-	def tearDown(self): SetConnections.CloseConnections()
 
 
 if __name__ == "__main__": unittest.main()

trunk/rdiff-backup/testing/regressiontest.py

 import unittest, os
 
 execfile("commontest.py")
-rbexec("setconnections.py")
+rbexec("main.py")
 
 
 """Regression tests
@@ -128,27 +128,28 @@ class PathSetter(unittest.TestCase):
 		SetConnections.CloseConnections()
 
 
-class IncrementTest(PathSetter):
+class IncrementTest1(unittest.TestCase):
+	dirlist = ["testfiles/increment1", "testfiles/increment2",
+			   "testfiles/increment3", "testfiles/increment4"]
+
 	def testLocalinc(self):
 		"""Test self.incrementing, and then restoring, local"""
-		self.setPathnames(None, None, None, None)
-		self.runtest()
+		BackupRestoreSeries(1, 1, self.dirlist)
 		
 	def test_remote_src(self):
 		"""Increment/Restore when source directory is remote"""
-		self.setPathnames('test1', '../', None, None)
-		self.runtest()
+		BackupRestoreSeries(None, 1, self.dirlist)
 		
 	def test_remote_dest(self):
 		"""Increment/Restore when target directory is remote"""
-		self.setPathnames(None, None, 'test2', '../')
-		self.runtest()
+		BackupRestoreSeries(1, None, self.dirlist)		
 		
 	def test_remote_both(self):
 		"""Increment/Restore when both directories are remote"""
-		self.setPathnames('test1', '../', 'test2/tmp', '../../')
-		self.runtest()
-		
+		BackupRestoreSeries(None, None, self.dirlist)
+
+
+class IncrementTest2(PathSetter):
 	def OldtestRecoveryLocal(self):
 		"""Test to see if rdiff-backup can continue with bad increment"""
 		os.system(MiscDir+'/myrm testfiles/recovery_out_backup')
@@ -188,43 +189,43 @@ class IncrementTest(PathSetter):
 		Time.setcurtime()
 		SaveState.init_filenames(1)
 		HighLevel.Mirror(self.inc1rp, self.rpout)
-		assert RPath.cmp_recursive(Local.inc1rp, Local.rpout)
+		assert CompareRecursive(Local.inc1rp, Local.rpout)
 
 		Time.setcurtime()
 		Time.setprevtime(999500000)
 		HighLevel.Mirror_and_increment(self.inc2rp, self.rpout, self.rpout_inc)
-		assert RPath.cmp_recursive(Local.inc2rp, Local.rpout)
+		assert CompareRecursive(Local.inc2rp, Local.rpout)
 
 		Time.setcurtime()
 		Time.setprevtime(999510000)
 		HighLevel.Mirror_and_increment(self.inc3rp, self.rpout, self.rpout_inc)
-		assert RPath.cmp_recursive(Local.inc3rp, Local.rpout)
+		assert CompareRecursive(Local.inc3rp, Local.rpout)
 
 		Time.setcurtime()
 		Time.setprevtime(999520000)
 		HighLevel.Mirror_and_increment(self.inc4rp, self.rpout, self.rpout_inc)
-		assert RPath.cmp_recursive(Local.inc4rp, Local.rpout)
+		assert CompareRecursive(Local.inc4rp, Local.rpout)
 		
 
 		print "Restoring to self.inc4"
 		HighLevel.Restore(999530000, self.rpout, self.get_inctup(),
 						  self.rpout4)
-		assert RPath.cmp_recursive(Local.inc4rp, Local.rpout4)
+		assert CompareRecursive(Local.inc4rp, Local.rpout4)
 
 		print "Restoring to self.inc3"
 		HighLevel.Restore(999520000, self.rpout, self.get_inctup(),
 						  self.rpout3)
-		assert RPath.cmp_recursive(Local.inc3rp, Local.rpout3)
+		assert CompareRecursive(Local.inc3rp, Local.rpout3)
 
 		print "Restoring to self.inc2"
 		HighLevel.Restore(999510000, self.rpout, self.get_inctup(),
 						  self.rpout2)
-		assert RPath.cmp_recursive(Local.inc2rp, Local.rpout2)
+		assert CompareRecursive(Local.inc2rp, Local.rpout2)
 
 		print "Restoring to self.inc1"
 		HighLevel.Restore(999500000, self.rpout, self.get_inctup(),
 						  self.rpout1)
-		assert RPath.cmp_recursive(Local.inc1rp, Local.rpout1)
+		assert CompareRecursive(Local.inc1rp, Local.rpout1)
 
 	def get_inctup(self):
 		"""Return inc tuples as expected by Restore.RestoreRecursive
@@ -288,7 +289,7 @@ class MirrorTest(PathSetter):
 		SaveState.init_filenames(None)
 		HighLevel.Mirror(self.noperms, self.noperms_out, None)
 		# Can't compare because we don't have the permissions to do it right
-		#assert RPath.cmp_recursive(Local.noperms, Local.noperms_out)
+		#assert CompareRecursive(Local.noperms, Local.noperms_out)
 
 	def testNopermsRemote(self):
 		"No permissions mirroring (remote)"
@@ -296,7 +297,7 @@ class MirrorTest(PathSetter):
 		Time.setcurtime()
 		SaveState.init_filenames(None)
 		HighLevel.Mirror(self.noperms, self.noperms_out, checkpoint=None)
-		#assert RPath.cmp_recursive(Local.noperms, Local.noperms_out)
+		#assert CompareRecursive(Local.noperms, Local.noperms_out)
 
 	def testPermSkipLocal(self):
 		"""Test to see if rdiff-backup will skip unreadable files"""
@@ -330,7 +331,7 @@ class MirrorTest(PathSetter):
 		self.refresh(self.rootfiles, self.rootfiles_out,
 				Local.rootfiles, Local.rootfiles_out) # add uid/gid info
 		HighLevel.Mirror(self.rootfiles, self.rootfiles_out)
-		assert RPath.cmp_recursive(Local.rootfiles, Local.rootfiles_out)
+		assert CompareRecursive(Local.rootfiles, Local.rootfiles_out)
 		Globals.change_ownership = None
 		self.refresh(self.rootfiles, self.rootfiles_out,
 				Local.rootfiles, Local.rootfiles_out) # remove that info
@@ -343,7 +344,7 @@ class MirrorTest(PathSetter):
 		self.refresh(self.rootfiles, self.rootfiles_out,
 				Local.rootfiles, Local.rootfiles_out) # add uid/gid info
 		HighLevel.Mirror(self.rootfiles, self.rootfiles_out)
-		assert RPath.cmp_recursive(Local.rootfiles, Local.rootfiles_out)
+		assert CompareRecursive(Local.rootfiles, Local.rootfiles_out)
 		for coon in Globals.connections:
 			conn.Globals.set('change_ownership', None)
 		self.refresh(self.rootfiles, self.rootfiles_out,
@@ -356,11 +357,11 @@ class MirrorTest(PathSetter):
 		self.refresh(self.rootfiles2, self.rootfiles_out2,
 				Local.rootfiles2, Local.rootfiles_out2) # add uid/gid info
 		HighLevel.Mirror(self.rootfiles2, self.rootfiles_out2)
-		assert RPath.cmp_recursive(Local.rootfiles2, Local.rootfiles_out2)
+		assert CompareRecursive(Local.rootfiles2, Local.rootfiles_out2)
 		self.refresh(self.rootfiles2, self.rootfiles_out2,
 				Local.rootfiles2, Local.rootfiles_out2) # remove that info
 		HighLevel.Mirror(self.rootfiles21, self.rootfiles_out2)
-		assert RPath.cmp_recursive(Local.rootfiles21, Local.rootfiles_out2)
+		assert CompareRecursive(Local.rootfiles21, Local.rootfiles_out2)
 		self.refresh(self.rootfiles21, self.rootfiles_out2,
 				Local.rootfiles21, Local.rootfiles_out2) # remove that info
 		Globals.change_source_perms = 1
@@ -387,12 +388,12 @@ class MirrorTest(PathSetter):
 		SaveState.init_filenames(None)
 		assert self.rbdir.lstat()
 		HighLevel.Mirror(self.inc1rp, self.rpout)
-		assert RPath.cmp_recursive(Local.inc1rp, Local.rpout)
+		assert CompareRecursive(Local.inc1rp, Local.rpout)
 
 		self.deleteoutput()
 
 		HighLevel.Mirror(self.inc2rp, self.rpout)
-		assert RPath.cmp_recursive(Local.inc2rp, Local.rpout)
+		assert CompareRecursive(Local.inc2rp, Local.rpout)
 
 	def run_partial_test(self):
 		os.system("cp -a testfiles/increment3 testfiles/output")
@@ -402,9 +403,9 @@ class MirrorTest(PathSetter):
 		SaveState.init_filenames(None)		
 		HighLevel.Mirror(self.inc1rp, self.rpout)
 		#RPath.copy_attribs(self.inc1rp, self.rpout)
-		assert RPath.cmp_recursive(Local.inc1rp, Local.rpout)
+		assert CompareRecursive(Local.inc1rp, Local.rpout)
 
 		HighLevel.Mirror(self.inc2rp, self.rpout)
-		assert RPath.cmp_recursive(Local.inc2rp, Local.rpout)
+		assert CompareRecursive(Local.inc2rp, Local.rpout)
 
 if __name__ == "__main__": unittest.main()

trunk/rdiff-backup/testing/roottest.py

@@ -136,7 +136,7 @@ class MirrorTest(PathSetter):
 		self.refresh(self.rootfiles, self.rootfiles_out,
 				Local.rootfiles, Local.rootfiles_out) # add uid/gid info
 		HighLevel.Mirror(self.rootfiles, self.rootfiles_out)
-		assert RPath.cmp_recursive(Local.rootfiles, Local.rootfiles_out)
+		assert CompareRecursive(Local.rootfiles, Local.rootfiles_out)
 		Globals.change_ownership = None
 		self.refresh(self.rootfiles, self.rootfiles_out,
 				Local.rootfiles, Local.rootfiles_out) # remove that info
@@ -151,7 +151,7 @@ class MirrorTest(PathSetter):
 		self.refresh(self.rootfiles, self.rootfiles_out,
 				Local.rootfiles, Local.rootfiles_out) # add uid/gid info
 		HighLevel.Mirror(self.rootfiles, self.rootfiles_out)
-		assert RPath.cmp_recursive(Local.rootfiles, Local.rootfiles_out)
+		assert CompareRecursive(Local.rootfiles, Local.rootfiles_out)
 		for coon in Globals.connections:
 			conn.Globals.set('change_ownership', None)
 		self.refresh(self.rootfiles, self.rootfiles_out,

trunk/rdiff-backup/testing/rorpitertest.py

@@ -62,25 +62,25 @@ class RORPIterTest(unittest.TestCase):
 			RORPIter.PatchIter(self.output, RORPIter.FromFile(diff_file))
 
 		turninto(self.inc1rp)
-		assert self.compare_rps(self.output, self.inc1rp)
+		RPath.copy_attribs(self.inc1rp, self.output) # Update time
+		assert self.compare_no_times(self.inc1rp, self.output)
 		turninto(self.inc2rp)
-		assert self.compare_rps(self.output, self.inc2rp)
-
-	def compare_rps(self, rp1, rp2):
-		"""True if rp1 and rp2 are equal in some sense"""
-		def RawIter(rp):
-			"""Get raw iterator of file stats based an rp1"""
-			return RORPIter.ToRaw(Iter.map(lambda rp2: rp2.getRORPath(),
-										   RORPIter.IterateRPaths(rp)))
-		ri1 = RawIter(rp1)
-		ri2 = RawIter(rp2)
-		try:
-			rorp1 = ri1.next()
-			rorp2 = ri2.next()
-			assert rorp1 == rorp2, "%s %s" % (rorp1, rorp2)
-		except StopIteration: pass
-		return 1
-	#	return Iter.equal(RawIter(rp1), RawIter(rp2))
+		RPath.copy_attribs(self.inc2rp, self.output)
+		assert self.compare_no_times(self.inc2rp, self.output)
+
+	def compare_no_times(self, src_rp, dest_rp):
+		"""Compare but disregard directories attributes"""
+		dsiter1, dsiter2 = map(DestructiveStepping.Iterate_with_Finalizer,
+							   [src_rp, dest_rp], [1, None])
+
+		def equal(src_rorp, dest_rorp):
+			return ((src_rorp.isdir() and dest_rorp.isdir()) or
+					src_rorp == dest_rorp)
+
+		result = Iter.equal(dsiter1, dsiter2, 1, equal)
+		for i in dsiter1: pass # make sure all files processed anyway
+		for i in dsiter2: pass
+		return result
 
 
 class IndexedTupleTest(unittest.TestCase):

trunk/rdiff-backup/testing/server.py

 #!/usr/bin/env python
 
-import sys
-execfile("commontest.py")
-rbexec("setconnections.py")
+import sys, os
 
-def Test_SetConnGlobals(conn, name, val):
-	"""Used in unittesting - set one of specified connection's Global vars"""
-	conn.Globals.set(name, val)
+__doc__ = """
 
-Log.setverbosity(9)
+This starts an rdiff-backup server using the existing source files.
+If not run from the source directory, the only argument should be 
+the directory the source files are in.
+"""
+
+def print_usage():
+	print "Usage: server.py  [path to source files]", __doc__
+
+if len(sys.argv) > 2:
+	print_usage()
+	sys.exit(1)
+
+try:
+	if len(sys.argv) == 2:
+		olddir = os.getcwd()
+		os.chdir(sys.argv[1])
+	execfile("setconnections.py")
+	if len(sys.argv) == 2: os.chdir(olddir)
+except (OSError, IOError):
+	print_usage()
+	raise
+
+#Log.setverbosity(9)
 PipeConnection(sys.stdin, sys.stdout).Server()