Skip to content

R
rdiff-backup
Commit fe98b295 authored by bescoto's avatar bescoto
Browse files
Options

Various changes for v0.11.1 (see CHANGELOG) 


git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@256 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109
parent d551bdbe
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 1499 additions and 2392 deletions
trunk/rdiff-backup/CHANGELOG
View file @ fe98b295
New in v0.11.1 (2002/12/??)
New in v0.11.1 (2002/12/31)
---------------------------
**Warning** Various features have been removed from this version, so
......@@ -27,8 +27,15 @@ The following features have been removed:
still generated, the directory statistics file no longer is,
because the new code structure makes it less inconvenient.
The various --exclude and --include options no longer work when
restoring. This may be added later if there is demand.
--windows-mode and filename quoting doesn't work. There have been
several requests for this in the past, so it will probably be
re-added in the next version.
Extensive refactoring. A lot of rdiff-backup's code was structured as
if it was still in one file, so it didn't make enough use of Python's
if it were still in one file, so it didn't make enough use of Python's
module system.
Now rdiff-backup writes metadata (uid, gid, mtime, etc.) to a
......@@ -42,9 +49,13 @@ some ramifications:
metadata, so it may not be necessary to traverse the whole mirror
directory. This can reduce file access on the destination side.
Even when the --no-hard-links option is given when backing up,
link relationships can be restored properly. However, if this
option is given, mirror files will not be linked together.
Special file types like device and sockets which cannot be created
on the remote side for some reason can still be backed up and
restored properly.
Fixed bug with the --{include|exclude}-globbing-filelist options
(reported by Claus Herwig).
......@@ -54,7 +65,8 @@ given date, and added Bud Bruegger's patch to that. The format and
information this option provides will probably change in the near
future.
Restoring is now pipelined for better high latency performance, and
unchanged files in the target directory will not be recopied.
New in v0.11.0 (2002/10/05)
......
trunk/rdiff-backup/dist/makedist
View file @ fe98b295
......@@ -2,7 +2,7 @@
import os, re, shutil, time, sys, getopt
SourceDir = "src"
SourceDir = "rdiff_backup"
DistDir = "dist"
# Various details about the files must also be specified by the rpm
......@@ -89,15 +89,15 @@ def MakeTar():
assert not os.system("cp %s %s" % (filename, tardir)), filename
os.mkdir(tardir+"/rdiff_backup")
for filename in ["connection.py", "destructive_stepping.py",
for filename in ["backup.py", "connection.py",
"FilenameMapping.py", "Hardlink.py",
"highlevel.py", "increment.py", "__init__.py",
"iterfile.py", "lazy.py", "librsync.py",
"log.py", "Main.py", "manage.py", "MiscStats.py",
"Rdiff.py", "restore.py", "rlist.py",
"robust.py", "rorpiter.py", "rpath.py", "Security.py",
"selection.py", "SetConnections.py", "static.py",
"statistics.py", "Time.py"]:
"increment.py", "__init__.py", "iterfile.py",
"lazy.py", "librsync.py", "log.py", "Main.py",
"manage.py", "metadata.py", "Rdiff.py",
"restore.py", "robust.py", "rorpiter.py",
"rpath.py", "Security.py", "selection.py",
"SetConnections.py", "static.py",
"statistics.py", "TempFile.py", "Time.py"]:
assert not os.system("cp %s/%s %s/rdiff_backup" %
(SourceDir, filename, tardir)), filename
......@@ -137,10 +137,10 @@ def parse_cmdline(arglist):
def Main():
action = parse_cmdline(sys.argv[1:])
if action == "FAQ":
print "Making FAQ"
MakeFAQ()
else:
print "Making FAQ"
MakeFAQ()
if action != "FAQ":
assert action == "All"
print "Processing version " + Version
tarfile = MakeTar()
......
trunk/rdiff-backup/dist/makerpm
View file @ fe98b295
......@@ -2,7 +2,7 @@
import os, sys, re
SourceDir = "src"
rpmroot = "/home/ben/rpm"
if len(sys.argv) == 2:
version = sys.argv[1]
......@@ -20,15 +20,9 @@ tarfile = "-".join(base.split("-")[:-1]) + ".tar.gz"
# These assume the rpm root directory $HOME/rpm. The
# nonstandard location allows for building by non-root user.
assert not os.system("cp %s $HOME/rpm/SOURCES" % (tarfile,))
assert not os.system("rpm -ba --sign -vv --target i386 " + specfile)
assert not os.system("cp $HOME/rpm/RPMS/i386/%s ." % i386rpm)
assert not os.system("cp $HOME/rpm/SRPMS/%s ." % source_rpm)
assert not os.system("cp %s %s/SOURCES" % (tarfile, rpmroot))
#assert not os.system("rpm -ba --sign -vv --target i386 " + specfile)
assert not os.system("rpmbuild -ba -v " + specfile)
assert not os.system("cp %s/RPMS/i386/%s ." % (rpmroot, i386rpm))
assert not os.system("cp %s/SRPMS/%s ." % (rpmroot, source_rpm))
# Old root RPM instructions
#assert not os.system("install -o root -g root -m 644 %s "
# "/usr/src/redhat/SOURCES" % (tarfile,))
#assert not os.system("install -o ben -g ben -m 644 "
# "/usr/src/redhat/RPMS/i386/%s ." % i386rpm)
#assert not os.system("install -o ben -g ben -m 644 "
# "/usr/src/redhat/SRPMS/%s ." % source_rpm)
trunk/rdiff-backup/rdiff-backup.1
View file @ fe98b295
......@@ -217,16 +217,12 @@ The default is
"(?i).*\\.(gz|z|bz|bz2|tgz|zip|rpm|deb|jpg|gif|png|jp2|mp3|ogg|avi|wmv|mpeg|mpg|rm|mov)$"
.TP
.BI --no-hard-links
Don't preserve hard links from source to mirror directories.
Otherwise, no increment files will themselves be hard linked, but a
hard link database will be written so that hard links from any dataset
will be recreated if originally present. If many hard linked files
are present, this option can drastically decrease memory usage.
.TP
.B --no-resume
Do not resume last aborted backup even if it falls within the resume
window.
.TP
Don't replicate hard links on destination side. Note that because
metadata is written to a separate file, hard link information will not
be lost even if the --no-hard-links option is given (however, mirror
files will not be linked). If many hard-linked files are present,
this option can drastically increase memory usage.
..TP
.B --null-separator
Use nulls (\\0) instead of newlines (\\n) as line separators, which
may help when dealing with filenames containing newlines. This
......
trunk/rdiff-backup/rdiff_backup/Hardlink.py
View file @ fe98b295
......@@ -21,12 +21,12 @@
If the preserve_hardlinks option is selected, linked files in the
source directory will be linked in the mirror directory. Linked files
are treated like any other with respect to incrementing, but a
database of all links will be recorded at each session, so linked
files can still be restored from the increments.
are treated like any other with respect to incrementing, but their
link status can be retrieved because their device location and inode #
is written in the metadata file.
All these functions are meant to be executed on the destination
side. The source side should only transmit inode information.
All these functions are meant to be executed on the mirror side. The
source side should only transmit inode information.
"""
......@@ -67,58 +67,6 @@ def clear_dictionaries():
_src_index_indicies = _dest_index_indicies = _restore_index_path = None
# The keys of this dictionary are (inode, devloc) pairs on the source
# side. The values are (numlinks, index) pairs, where numlinks are
# the number of files currently linked to this spot, and index is the
# index of the first file so linked.
_src_inode_index_dict = {}
_dest_inode_index_dict = {}
#def rorp_eq(src_rorp, dest_rorp):
# """Return true if source and dest rorp are equal as far as hardlinking
#
# This also processes the src_rorp, adding it if necessary to the
# inode dictionary.
#
# """
# if not src_rorp.isreg(): return 1 # only reg files can be hard linked
# if src_rorp.getnumlinks() == 1: return dest_rorp.getnumlinks() == 1
#
# src_linked_index = process_rorp(src_rorp, _src_inode_index_dict)
# if dest_rorp.getnumlinks() == 1: return 0
# dest_linked_index = process_rorp(dest_rorp, _dest_inode_index_dict)
# return src_linked_index == dest_linked_index
def process_rorp(rorp, inode_dict):
"""Add inode info and returns index src_rorp is linked to, or None"""
key_pair = (rorp.getinode(), rorp.getdevloc())
try: num, linked_index = inode_dict[key_pair]
except KeyError:
inode_dict[key_pair] = (1, src_rorp.index)
return None
inode_dict[key_pair] = (num+1, linked_index)
if num+1 == src_rorp.getnumlinks(): del _inode_index_dict[key_pair]
else: _inode_index_dict[key_pair] = (num+1, linked_index)
return linked_index
def get_linked_index(src_rorp):
"""Return the index a src_rorp is linked to, or None
Also deletes the src_rorp's entry in the dictionary if we have
accumulated all the hard link references.
"""
key_pair = (rorp.getinode(), rorp.getdevloc())
try: num, linked_index = _src_inode_index_dict[key_pair]
except KeyError: return None
if num == src_rorp.getnumlinks():
del _src_inode_index_dict[key_pair]
def get_inode_key(rorp):
"""Return rorp's key for _inode_ dictionaries"""
return (rorp.getinode(), rorp.getdevloc())
......@@ -190,6 +138,10 @@ def islinked(rorp):
"""True if rorp's index is already linked to something on src side"""
return len(get_indicies(rorp, 1)) >= 2
def get_link_index(rorp):
"""Return first index on target side rorp is already linked to"""
return get_indicies(rorp, 1)[0]
def restore_link(index, rpath):
"""Restores a linked file by linking it
......@@ -214,129 +166,13 @@ def restore_link(index, rpath):
_restore_index_path[index] = rpath.path
return None
def link_rp(src_rorp, dest_rpath, dest_root = None):
"""Make dest_rpath into a link analogous to that of src_rorp"""
def link_rp(diff_rorp, dest_rpath, dest_root = None):
"""Make dest_rpath into a link using link flag in diff_rorp"""
if not dest_root: dest_root = dest_rpath # use base of dest_rpath
dest_link_rpath = rpath.RPath(dest_root.conn, dest_root.base,
get_indicies(src_rorp, 1)[0])
diff_rorp.get_link_flag())
dest_rpath.hardlink(dest_link_rpath.path)
def write_linkdict(rpath, dict, compress = None):
"""Write link data to the rbdata dir
It is stored as the a big pickled dictionary dated to match
the current hardlinks.
"""
assert (Globals.isbackup_writer and
rpath.conn is Globals.local_connection)
tf = TempFile.new(rpath)
def init():
fp = tf.open("wb", compress)
cPickle.dump(dict, fp)
assert not fp.close()
tf.setdata()
robust.make_tf_robustaction(init, (tf,), (rpath,)).execute()
def get_linkrp(data_rpath, time, prefix):
"""Return RPath of linkdata, or None if cannot find"""
for rp in map(data_rpath.append, data_rpath.listdir()):
if (rp.isincfile() and rp.getincbase_str() == prefix and
(rp.getinctype() == 'snapshot' or rp.getinctype() == 'data')
and Time.stringtotime(rp.getinctime()) == time):
return rp
return None
def get_linkdata(data_rpath, time, prefix = 'hardlink_data'):
"""Return index dictionary written by write_linkdata at time"""
rp = get_linkrp(data_rpath, time, prefix)
if not rp: return None
fp = rp.open("rb", rp.isinccompressed())
index_dict = cPickle.load(fp)
assert not fp.close()
return index_dict
def final_writedata():
"""Write final checkpoint data to rbdir after successful backup"""
global final_inc
if _src_index_indicies:
log.Log("Writing hard link data", 6)
if Globals.compression:
final_inc = Globals.rbdir.append("hardlink_data.%s.data.gz" %
Time.curtimestr)
else: final_inc = Globals.rbdir.append("hardlink_data.%s.data" %
Time.curtimestr)
write_linkdict(final_inc, _src_index_indicies, Globals.compression)
else: # no hardlinks, so writing unnecessary
final_inc = None
def retrieve_final(time):
"""Set source index dictionary from hardlink_data file if avail"""
global _src_index_indicies
hd = get_linkdata(Globals.rbdir, time)
if hd is None: return None
_src_index_indicies = hd
return 1
def final_checkpoint(data_rpath):
"""Write contents of the four dictionaries to the data dir
If rdiff-backup receives a fatal error, it may still be able
to save the contents of the four hard link dictionaries.
Because these dictionaries may be big, they are not saved
after every 20 seconds or whatever, but just at the end.
"""
log.Log("Writing intermediate hard link data to disk", 2)
src_inode_rp = data_rpath.append("hardlink_source_inode_checkpoint."
"%s.data" % Time.curtimestr)
src_index_rp = data_rpath.append("hardlink_source_index_checkpoint."
"%s.data" % Time.curtimestr)
dest_inode_rp = data_rpath.append("hardlink_dest_inode_checkpoint."
"%s.data" % Time.curtimestr)
dest_index_rp = data_rpath.append("hardlink_dest_index_checkpoint."
"%s.data" % Time.curtimestr)
for (rp, dict) in ((src_inode_rp, _src_inode_indicies),
(src_index_rp, _src_index_indicies),
(dest_inode_rp, _dest_inode_indicies),
(dest_index_rp, _dest_index_indicies)):
write_linkdict(rp, dict)
def retrieve_checkpoint(data_rpath, time):
"""Retrieve hardlink data from final checkpoint
Return true if the retrieval worked, false otherwise.
"""
global _src_inode_indicies, _src_index_indicies
global _dest_inode_indicies, _dest_index_indicies
try:
src_inode = get_linkdata(data_rpath, time,
"hardlink_source_inode_checkpoint")
src_index = get_linkdata(data_rpath, time,
"hardlink_source_index_checkpoint")
dest_inode = get_linkdata(data_rpath, time,
"hardlink_dest_inode_checkpoint")
dest_index = get_linkdata(data_rpath, time,
"hardlink_dest_index_checkpoint")
except cPickle.UnpicklingError:
log.Log("Unpickling Error", 2)
return None
if (src_inode is None or src_index is None or
dest_inode is None or dest_index is None): return None
_src_inode_indicies, _src_index_indicies = src_inode, src_index
_dest_inode_indicies, _dest_index_indicies = dest_inode, dest_index
return 1
def remove_all_checkpoints():
"""Remove all hardlink checkpoint information from directory"""
prefix_list = ["hardlink_source_inode_checkpoint",
"hardlink_source_index_checkpoint",
"hardlink_dest_inode_checkpoint",
"hardlink_dest_index_checkpoint"]
for rp in map(Globals.rbdir.append, Globals.rbdir.listdir()):
if (rp.isincfile() and rp.getincbase_str() in prefix_list and
(rp.getinctype() == 'snapshot' or rp.getinctype() == 'data')):
rp.delete()
trunk/rdiff-backup/rdiff_backup/Main.py
View file @ fe98b295
# Copyright 2002 Ben Escoto
#
# This file is part of rdiff-backup.
......@@ -21,9 +22,9 @@
from __future__ import generators
import getopt, sys, re, os
from log import Log
from log import Log, LoggerError
import Globals, Time, SetConnections, selection, robust, rpath, \
manage, highlevel, connection, restore, FilenameMapping, \
manage, backup, connection, restore, FilenameMapping, \
Security, Hardlink
......@@ -146,6 +147,7 @@ def parse_cmdlineoptions(arglist):
Globals.set('quoting_enabled', 1)
Globals.set('preserve_hardlinks', 0)
select_opts.append(("--exclude-special-files", None))
assert 0, "Windows mode doesn't work in this version!"
elif opt == '--windows-time-format':
Globals.set('time_separator', "_")
else: Log.FatalError("Unknown option %s" % opt)
......@@ -184,13 +186,6 @@ def commandline_error(message):
def misc_setup(rps):
"""Set default change ownership flag, umask, relay regexps"""
if ((len(rps) == 2 and rps[1].conn.os.getuid() == 0) or
(len(rps) < 2 and os.getuid() == 0)):
# Allow change_ownership if destination connection is root
for conn in Globals.connections:
conn.Globals.set('change_ownership', 1)
for rp in rps: rp.setdata() # Update with userinfo
os.umask(077)
Time.setcurtime(Globals.current_time)
FilenameMapping.set_init_quote_vals()
......@@ -240,14 +235,14 @@ def Backup(rpin, rpout):
backup_init_dirs(rpin, rpout)
if prevtime:
Time.setprevtime(prevtime)
highlevel.Mirror_and_increment(rpin, rpout, incdir)
else: highlevel.Mirror(rpin, rpout)
backup.Mirror_and_increment(rpin, rpout, incdir)
else: backup.Mirror(rpin, rpout)
rpout.conn.Main.backup_touch_curmirror_local(rpin, rpout)
def backup_set_select(rpin):
"""Create Select objects on source connection"""
rpin.conn.highlevel.HLSourceStruct.set_source_select(rpin, select_opts,
*select_files)
rpin.conn.backup.SourceStruct.set_source_select(rpin, select_opts,
*select_files)
def backup_init_dirs(rpin, rpout):
"""Make sure rpin and rpout are valid, init data dir and logging"""
......@@ -322,8 +317,7 @@ def backup_get_mirrortime():
"""Warning: duplicate current_mirror files found. Perhaps something
went wrong during your last backup? Using """ + mirrorrps[-1].path, 2)
timestr = mirrorrps[-1].getinctime()
return Time.stringtotime(timestr)
return mirrorrps[-1].getinctime()
def backup_touch_curmirror_local(rpin, rpout):
"""Make a file like current_mirror.time.data to record time
......@@ -348,8 +342,7 @@ def Restore(src_rp, dest_rp = None):
"""
rpin, rpout = restore_check_paths(src_rp, dest_rp)
time = Time.stringtotime(rpin.getinctime())
restore_common(rpin, rpout, time)
restore_common(rpin, rpout, rpin.getinctime())
def RestoreAsOf(rpin, target):
"""Secondary syntax for restore operation
......@@ -365,12 +358,14 @@ def RestoreAsOf(rpin, target):
def restore_common(rpin, target, time):
"""Restore operation common to Restore and RestoreAsOf"""
if target.conn.os.getuid() == 0:
SetConnections.UpdateGlobal('change_ownership', 1)
mirror_root, index = restore_get_root(rpin)
mirror = mirror_root.new_index(index)
inc_rpath = datadir.append_path('increments', index)
restore_init_select(mirror_root, target)
restore_start_log(rpin, target, time)
restore.Restore(inc_rpath, mirror, target, time)
restore.Restore(mirror, inc_rpath, target, time)
Log("Restore ended", 4)
def restore_start_log(rpin, target, time):
......@@ -397,9 +392,9 @@ Try restoring from an increment file (the filenames look like
if not rpout: rpout = rpath.RPath(Globals.local_connection,
rpin.getincbase_str())
if rpout.lstat():
if rpout.lstat() and not force:
Log.FatalError("Restore target %s already exists, "
"and will not be overwritten." % rpout.path)
"specify --force to overwrite." % rpout.path)
return rpin, rpout
def restore_init_select(rpin, rpout):
......@@ -462,15 +457,13 @@ def restore_get_root(rpin):
def ListIncrements(rp):
"""Print out a summary of the increments and their times"""
mirror_root, index = restore_get_root(rp)
Globals.rbdir = datadir = \
mirror_root.append_path("rdiff-backup-data")
mirrorrp = mirror_root.new_index(index)
inc_rpath = datadir.append_path('increments', index)
mirror_rp = mirror_root.new_index(index)
inc_rpath = Globals.rbdir.append_path('increments', index)
incs = restore.get_inclist(inc_rpath)
mirror_time = restore.get_mirror_time()
mirror_time = restore.MirrorStruct.get_mirror_time()
if Globals.parsable_output:
print manage.describe_incs_parsable(incs, mirror_time, mirrorrp)
else: print manage.describe_incs_human(incs, mirror_time, mirrorrp)
print manage.describe_incs_parsable(incs, mirror_time, mirror_rp)
else: print manage.describe_incs_human(incs, mirror_time, mirror_rp)
def CalculateAverage(rps):
......@@ -493,8 +486,8 @@ def RemoveOlderThan(rootrp):
timep = Time.timetopretty(time)
Log("Deleting increment(s) before %s" % timep, 4)
times_in_secs = map(lambda inc: Time.stringtotime(inc.getinctime()),
restore.get_inclist(datadir.append("increments")))
times_in_secs = [inc.getinctime() for inc in
restore.get_inclist(datadir.append("increments"))]
times_in_secs = filter(lambda t: t < time, times_in_secs)
if not times_in_secs:
Log.FatalError("No increments older than %s found" % timep)
......@@ -517,30 +510,7 @@ def ListChangedSince(rp):
try: rest_time = Time.genstrtotime(restore_timestr)
except Time.TimeException, exc: Log.FatalError(str(exc))
mirror_root, index = restore_get_root(rp)
Globals.rbdir = datadir = mirror_root.append_path("rdiff-backup-data")
mirror_time = restore.get_mirror_time()
def get_rids_recursive(rid):
"""Yield all the rids under rid that have inc newer than rest_time"""
yield rid
for sub_rid in restore.yield_rids(rid, rest_time, mirror_time):
for sub_sub_rid in get_rids_recursive(sub_rid): yield sub_sub_rid
def determineChangeType(incList):
"returns the type of change determined from incList"
assert len(incList) > 0
last_inc_type = incList[-1].getinctype() # examine earliest change
if last_inc_type == 'snapshot': return "misc change"
elif last_inc_type == 'missing': return "new file"
elif last_inc_type == 'diff': return "modified"
elif last_inc_type == 'dir': return "dir change"
else: return "Unknown!"
inc_rpath = datadir.append_path('increments', index)
inc_list = restore.get_inclist(inc_rpath)
root_rid = restore.RestoreIncrementData(index, inc_rpath, inc_list)
for rid in get_rids_recursive(root_rid):
if rid.inc_list:
print "%-11s: %s" % (determineChangeType(rid.inc_list),
rid.get_indexpath())
mirror_rp = mirror_root.new_index(index)
inc_rp = mirror_rp.append_path("increments", index)
restore.ListChangedSince(mirror_rp, inc_rp, rest_time)
trunk/rdiff-backup/rdiff_backup/Rdiff.py
View file @ fe98b295
......@@ -17,75 +17,53 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
# USA
"""Invoke rdiff utility to make signatures, deltas, or patch
All these operations should be done in a relatively safe manner using
RobustAction and the like.
"""
"""Invoke rdiff utility to make signatures, deltas, or patch"""
import os, librsync
from log import Log
import robust, TempFile, Globals
import Globals, log, static, TempFile, rpath
def get_signature(rp):
"""Take signature of rpin file and return in file object"""
Log("Getting signature of %s" % rp.path, 7)
log.Log("Getting signature of %s" % rp.path, 7)
return librsync.SigFile(rp.open("rb"))
def get_delta_sigfileobj(sig_fileobj, rp_new):
"""Like get_delta but signature is in a file object"""
Log("Getting delta of %s with signature stream" % (rp_new.path,), 7)
log.Log("Getting delta of %s with signature stream" % (rp_new.path,), 7)
return librsync.DeltaFile(sig_fileobj, rp_new.open("rb"))
def get_delta_sigrp(rp_signature, rp_new):
"""Take signature rp and new rp, return delta file object"""
Log("Getting delta of %s with signature %s" %
(rp_new.path, rp_signature.get_indexpath()), 7)
log.Log("Getting delta of %s with signature %s" %
(rp_new.path, rp_signature.get_indexpath()), 7)
return librsync.DeltaFile(rp_signature.open("rb"), rp_new.open("rb"))
def write_delta_action(basis, new, delta, compress = None):
"""Return action writing delta which brings basis to new
If compress is true, the output of rdiff will be gzipped
before written to delta.
"""
delta_tf = TempFile.new(delta)
def init(): write_delta(basis, new, delta_tf, compress)
return robust.make_tf_robustaction(init, delta_tf, delta)
def write_delta(basis, new, delta, compress = None):
"""Write rdiff delta which brings basis to new"""
Log("Writing delta %s from %s -> %s" %
(basis.path, new.path, delta.path), 7)
log.Log("Writing delta %s from %s -> %s" %
(basis.path, new.path, delta.path), 7)
sigfile = librsync.SigFile(basis.open("rb"))
deltafile = librsync.DeltaFile(sigfile, new.open("rb"))
delta.write_from_fileobj(deltafile, compress)
def patch_action(rp_basis, rp_delta, rp_out = None, out_tf = None,
delta_compressed = None):
"""Return RobustAction which patches rp_basis with rp_delta
def write_patched_fp(basis_fp, delta_fp, out_fp):
"""Write patched file to out_fp given input fps. Closes input files"""
rpath.copyfileobj(librsync.PatchedFile(basis_fp, delta_fp), out_fp)
assert not basis_fp.close() and not delta_fp.close()
If rp_out is None, put output in rp_basis. Will use TempFile
out_tf it is specified. If delta_compressed is true, the
delta file will be decompressed before processing with rdiff.
def write_via_tempfile(fp, rp):
"""Write fileobj fp to rp by writing to tempfile and renaming"""
tf = TempFile.new(rp)
tf.write_from_fileobj(fp)
tf.rename(rp)
"""
if not rp_out: rp_out = rp_basis
if not out_tf: out_tf = TempFile.new(rp_out)
def init():
rp_basis.conn.Rdiff.patch_local(rp_basis, rp_delta,
out_tf, delta_compressed)
out_tf.setdata()
return robust.make_tf_robustaction(init, out_tf, rp_out)
def patch_local(rp_basis, rp_delta, outrp, delta_compressed = None):
"""Patch routine that must be run on rp_basis.conn
def patch_local(rp_basis, rp_delta, outrp = None, delta_compressed = None):
"""Patch routine that must be run locally, writes to outrp
This is because librsync may need to seek() around in rp_basis,
and so needs a real file. Other rpaths can be remote.
This should be run local to rp_basis because it needs to be a real
file (librsync may need to seek around in it). If outrp is None,
patch rp_basis instead.
"""
assert rp_basis.conn is Globals.local_connection
......@@ -94,32 +72,19 @@ def patch_local(rp_basis, rp_delta, outrp, delta_compressed = None):
sigfile = librsync.SigFile(rp_basis.open("rb"))
patchfile = librsync.PatchedFile(rp_basis.open("rb"), deltafile)
outrp.write_from_fileobj(patchfile)
def patch_with_attribs_action(rp_basis, rp_delta, rp_out = None):
"""Like patch_action, but also transfers attributs from rp_delta"""
if not rp_out: rp_out = rp_basis
tf = TempFile.new(rp_out)
return robust.chain_nested(patch_action(rp_basis, rp_delta, rp_out, tf),
robust.copy_attribs_action(rp_delta, tf))
def copy_action(rpin, rpout):
"""Use rdiff to copy rpin to rpout, conserving bandwidth"""
if not rpin.isreg() or not rpout.isreg() or rpin.conn is rpout.conn:
# rdiff not applicable, fallback to regular copying
return robust.copy_action(rpin, rpout)
Log("Rdiff copying %s to %s" % (rpin.path, rpout.path), 6)
out_tf = TempFile.new(rpout)
def init(): rpout.conn.Rdiff.copy_local(rpin, rpout, out_tf)
return robust.make_tf_robustaction(init, out_tf, rpout)
def copy_local(rpin, rpout, rpnew):
if outrp: outrp.write_from_fileobj(patchfile)
else: write_via_tempfile(patchfile, rp_basis)
def copy_local(rpin, rpout, rpnew = None):
"""Write rpnew == rpin using rpout as basis. rpout and rpnew local"""
assert rpnew.conn is rpout.conn is Globals.local_connection
assert rpout.conn is Globals.local_connection
sigfile = librsync.SigFile(rpout.open("rb"))
deltafile = rpin.conn.librsync.DeltaFile(sigfile, rpin.open("rb"))
rpnew.write_from_fileobj(librsync.PatchedFile(rpout.open("rb"), deltafile))
patched_file = librsync.PatchedFile(rpout.open("rb"), deltafile)
if rpnew: rpnew.write_from_fileobj(patched_file)
else: write_via_tempfile(patched_file, rpout)
trunk/rdiff-backup/rdiff_backup/Security.py
View file @ fe98b295
......@@ -112,6 +112,7 @@ def set_allowed_requests(sec_level):
"Globals.get_dict_val",
"log.Log.open_logfile_allconn",
"log.Log.close_logfile_allconn",
"Log.log_to_file",
"SetConnections.add_redirected_conn",
"RedirectedRun",
"sys.stdout.write"]
......@@ -123,20 +124,21 @@ def set_allowed_requests(sec_level):
"os.listdir",
"Time.setcurtime_local",
"robust.Resume.ResumeCheck",
"highlevel.HLSourceStruct.split_initial_dsiter",
"highlevel.HLSourceStruct.get_diffs_and_finalize",
"backup.SourceStruct.split_initial_dsiter",
"backup.SourceStruct.get_diffs_and_finalize",
"rpath.gzip_open_local_read",
"rpath.open_local_read"])
if sec_level == "update-only":
allowed_requests.extend(
["Log.open_logfile_local", "Log.close_logfile_local",
"Log.close_logfile_allconn", "Log.log_to_file",
"log.Log.log_to_file",
"robust.SaveState.init_filenames",
"robust.SaveState.touch_last_file",
"highlevel.HLDestinationStruct.get_sigs",
"highlevel.HLDestinationStruct.patch_w_datadir_writes",
"highlevel.HLDestinationStruct.patch_and_finalize",
"highlevel.HLDestinationStruct.patch_increment_and_finalize",
"backup.DestinationStruct.get_sigs",
"backup.DestinationStruct.patch_w_datadir_writes",
"backup.DestinationStruct.patch_and_finalize",
"backup.DestinationStruct.patch_increment_and_finalize",
"Main.backup_touch_curmirror_local",
"Globals.ITRB.increment_stat"])
if Globals.server:
......@@ -148,8 +150,8 @@ def set_allowed_requests(sec_level):
"FilenameMapping.set_init_quote_vals_local",
"Globals.postset_regexp_local",
"Globals.set_select",
"highlevel.HLSourceStruct.set_session_info",
"highlevel.HLDestinationStruct.set_session_info"])
"backup.SourceStruct.set_session_info",
"backup.DestinationStruct.set_session_info"])
def vet_request(request, arglist):
"""Examine request for security violations"""
......
trunk/rdiff-backup/rdiff_backup/SetConnections.py
View file @ fe98b295
......@@ -201,6 +201,7 @@ def BackupInitConnections(reading_conn, writing_conn):
writing_conn.Globals.set("isbackup_writer", 1)
UpdateGlobal("backup_reader", reading_conn)
UpdateGlobal("backup_writer", writing_conn)
if writing_conn.os.getuid() == 0: UpdateGlobal('change_ownership', 1)
def CloseConnections():
"""Close all connections. Run by client"""
......
trunk/rdiff-backup/rdiff_backup/Time.py
View file @ fe98b295
......@@ -40,7 +40,7 @@ def setcurtime(curtime = None):
"""Sets the current time in curtime and curtimestr on all systems"""
t = curtime or time.time()
for conn in Globals.connections:
conn.Time.setcurtime_local(t)
conn.Time.setcurtime_local(long(t))
def setcurtime_local(timeinseconds):
"""Only set the current time locally"""
......
trunk/rdiff-backup/rdiff_backup/backup.py
View file @ fe98b295
This diff is collapsed.
trunk/rdiff-backup/rdiff_backup/connection.py
View file @ fe98b295
......@@ -96,7 +96,6 @@ class LowLevelPipeConnection(Connection):
b - string
q - quit signal
t - TempFile
d - DSRPath
R - RPath
r - RORPath only
c - PipeConnection object
......@@ -124,8 +123,6 @@ class LowLevelPipeConnection(Connection):
elif isinstance(obj, connection.Connection):self._putconn(obj, req_num)
elif isinstance(obj, TempFile.TempFile):
self._puttempfile(obj, req_num)
elif isinstance(obj, destructive_stepping.DSRPath):
self._putdsrpath(obj, req_num)
elif isinstance(obj, rpath.RPath): self._putrpath(obj, req_num)
elif isinstance(obj, rpath.RORPath): self._putrorpath(obj, req_num)
elif ((hasattr(obj, "read") or hasattr(obj, "write"))
......@@ -157,11 +154,6 @@ class LowLevelPipeConnection(Connection):
tempfile.index, tempfile.data)
self._write("t", cPickle.dumps(tf_repr, 1), req_num)
def _putdsrpath(self, dsrpath, req_num):
"""Put DSRPath into pipe. See _putrpath"""
dsrpath_repr = (dsrpath.conn.conn_number, dsrpath.getstatedict())
self._write("d", cPickle.dumps(dsrpath_repr, 1), req_num)
def _putrpath(self, rpath, req_num):
"""Put an rpath into the pipe
......@@ -246,7 +238,6 @@ class LowLevelPipeConnection(Connection):
elif format_string == "t": result = self._gettempfile(data)
elif format_string == "r": result = self._getrorpath(data)
elif format_string == "R": result = self._getrpath(data)
elif format_string == "d": result = self._getdsrpath(data)
else:
assert format_string == "c", header_string
result = Globals.connection_dict[int(data)]
......@@ -270,16 +261,6 @@ class LowLevelPipeConnection(Connection):
return rpath.RPath(Globals.connection_dict[conn_number],
base, index, data)
def _getdsrpath(self, raw_dsrpath_buf):
"""Return DSRPath object indicated by buf"""
conn_number, state_dict = cPickle.loads(raw_dsrpath_buf)
empty_dsrp = destructive_stepping.DSRPath("bypass",
Globals.local_connection, None)
empty_dsrp.__setstate__(state_dict)
empty_dsrp.conn = Globals.connection_dict[conn_number]
empty_dsrp.file = None
return empty_dsrp
def _close(self):
"""Close the pipes associated with the connection"""
self.outpipe.close()
......@@ -544,8 +525,8 @@ class VirtualFile:
# put at bottom to reduce circularities.
import Globals, Time, Rdiff, Hardlink, FilenameMapping, C, Security, \
Main, rorpiter, selection, increment, statistics, manage, lazy, \
iterfile, rpath, robust, restore, manage, highlevel, connection, \
TempFile, destructive_stepping, SetConnections
iterfile, rpath, robust, restore, manage, backup, connection, \
TempFile, SetConnections, librsync
from log import Log
Globals.local_connection = LocalConnection()
......
trunk/rdiff-backup/rdiff_backup/manage.py
View file @ fe98b295
......@@ -56,7 +56,7 @@ def describe_incs_parsable(incs, mirror_time, mirrorrp):
50000 regular <- last will be the current mirror
"""
incpairs = [(Time.stringtotime(inc.getinctime()), inc) for inc in incs]
incpairs = [(inc.getinctime(), inc) for inc in incs]
incpairs.sort()
result = ["%s %s" % (time, get_inc_type(inc)) for time, inc in incpairs]
result.append("%s %s" % (mirror_time, get_file_type(mirrorrp)))
......@@ -64,7 +64,7 @@ def describe_incs_parsable(incs, mirror_time, mirrorrp):
def describe_incs_human(incs, mirror_time, mirrorrp):
"""Return a string describing all the the root increments"""
incpairs = [(Time.stringtotime(inc.getinctime()), inc) for inc in incs]
incpairs = [(inc.getinctime(), inc) for inc in incs]
incpairs.sort()
result = ["Found %d increments:" % len(incpairs)]
......@@ -95,8 +95,7 @@ def delete_earlier_than_local(baserp, time):
yield sub_rp
for rp in yield_files(baserp):
if ((rp.isincfile() and
Time.stringtotime(rp.getinctime()) < time) or
if ((rp.isincfile() and rp.getinctime() < time) or
(rp.isdir() and not rp.listdir())):
Log("Deleting increment file %s" % rp.path, 5)
rp.delete()
......@@ -114,7 +113,7 @@ class IncObj:
if not incrp.isincfile():
raise ManageException("%s is not an inc file" % incrp.path)
self.incrp = incrp
self.time = Time.stringtotime(incrp.getinctime())
self.time = incrp.getinctime()
def getbaserp(self):
"""Return rp of the incrp without extensions"""
......
trunk/rdiff-backup/rdiff_backup/metadata.py
View file @ fe98b295
......@@ -56,7 +56,7 @@ field names and values.
from __future__ import generators
import re, gzip
import log, Globals, rpath, Time, robust
import log, Globals, rpath, Time, robust, increment
class ParsingError(Exception):
"""This is raised when bad or unparsable data is received"""
......@@ -207,6 +207,7 @@ class rorp_extractor:
log.Log("Error parsing metadata file: %s" % (e,), 2)
if self.at_end: break
self.buf = self.buf[next_pos:]
assert not self.close()
def skip_to_index(self, index):
"""Scan through the file, set buffer to beginning of index record
......@@ -250,6 +251,7 @@ class rorp_extractor:
yield rorp
if self.at_end: break
self.buf = self.buf[next_pos:]
assert not self.close()
def close(self):
"""Return value of closing associated file"""
......@@ -264,9 +266,10 @@ def OpenMetadata(rp = None, compress = 1):
assert not metadata_fileobj, "Metadata file already open"
if rp: metadata_rp = rp
else:
if compress: filename_base = "mirror_metadata.%s.data.gz"
else: filename_base = "mirror_metadata.%s.data"
metadata_rp = Globals.rbdir.append(filename_base % (Time.curtimestr,))
if compress: typestr = 'data.gz'
else: typestr = 'data'
metadata_rp = Globals.rbdir.append("mirror_metadata.%s.%s" %
(Time.curtimestr, typestr))
metadata_fileobj = metadata_rp.open("wb", compress = compress)
def WriteMetadata(rorp):
......@@ -307,8 +310,7 @@ def GetMetadata_at_time(rbdir, time, restrict_index = None, rblist = None):
for rp in rblist:
if (rp.isincfile() and rp.getinctype() == "data" and
rp.getincbase_str() == "mirror_metadata"):
if Time.stringtotime(rp.getinctime()) == time:
return GetMetadata(rp, restrict_index)
if rp.getinctime() == time: return GetMetadata(rp, restrict_index)
return None
trunk/rdiff-backup/rdiff_backup/restore.py
View file @ fe98b295
This diff is collapsed.
trunk/rdiff-backup/rdiff_backup/robust.py
View file @ fe98b295
This diff is collapsed.
trunk/rdiff-backup/rdiff_backup/rorpiter.py
View file @ fe98b295
......@@ -53,6 +53,12 @@ def FromRaw(raw_iter):
rorp.setfile(getnext(raw_iter))
yield rorp
def getnext(iter):
"""Return the next element of an iterator, raising error if none"""
try: next = iter.next()
except StopIteration: raise RORPIterException("Unexpected end to iter")
return next
def ToFile(rorp_iter):
"""Return file version of iterator"""
return iterfile.FileWrappingIter(ToRaw(rorp_iter))
......@@ -143,90 +149,23 @@ def Collate2Iters(riter1, riter2):
yield (None, relem2)
relem2 = None
def getnext(iter):
"""Return the next element of an iterator, raising error if none"""
try: next = iter.next()
except StopIteration: raise RORPIterException("Unexpected end to iter")
return next
def get_dissimilar_indicies(src_init_iter, dest_init_iter):
def get_dissimilar_indicies(src_init_iter, dest_init_iter, statfileobj = None):
"""Get dissimilar indicies given two rorpiters
Returns an iterator which enumerates the indicies of the rorps
which are different on the source and destination ends.
which are different on the source and destination ends. If
statfileobj is given, call add_changed on each pair of different
indicies.
"""
collated = Collate2Iters(src_init_iter, dest_init_iter)
for src_rorp, dest_rorp in collated:
if not src_rorp: yield dest_rorp.index
elif not dest_rorp: yield src_rorp.index
elif not src_rorp == dest_rorp: yield dest_rorp.index
elif (Globals.preserve_hardlinks and not
Hardlink.rorp_eq(src_rorp, dest_rorp)): yield dest_rorp.index
def GetDiffIter(sig_iter, new_iter):
"""Return delta iterator from sig_iter to new_iter
The accompanying file for each will be a delta as produced by
rdiff, unless the destination file does not exist, in which
case it will be the file in its entirety.
sig_iter may be composed of rorps, but new_iter should have
full RPaths.
"""
collated_iter = CollateIterators(sig_iter, new_iter)
for rorp, rp in collated_iter: yield diffonce(rorp, rp)
def diffonce(sig_rorp, new_rp):
"""Return one diff rorp, based from signature rorp and orig rp"""
if sig_rorp and Globals.preserve_hardlinks and sig_rorp.isflaglinked():
if new_rp: diff_rorp = new_rp.getRORPath()
else: diff_rorp = rpath.RORPath(sig_rorp.index)
diff_rorp.flaglinked()
return diff_rorp
elif sig_rorp and sig_rorp.isreg() and new_rp and new_rp.isreg():
diff_rorp = new_rp.getRORPath()
#fp = sig_rorp.open("rb")
#print "---------------------", fp
#tmp_sig_rp = RPath(Globals.local_connection, "/tmp/sig")
#tmp_sig_rp.delete()
#tmp_sig_rp.write_from_fileobj(fp)
#diff_rorp.setfile(Rdiff.get_delta_sigfileobj(tmp_sig_rp.open("rb"),
# new_rp))
diff_rorp.setfile(Rdiff.get_delta_sigfileobj(sig_rorp.open("rb"),
new_rp))
diff_rorp.set_attached_filetype('diff')
return diff_rorp
else:
# Just send over originial if diff isn't appropriate
if sig_rorp: sig_rorp.close_if_necessary()
if not new_rp: return rpath.RORPath(sig_rorp.index)
elif new_rp.isreg():
diff_rorp = new_rp.getRORPath(1)
diff_rorp.set_attached_filetype('snapshot')
return diff_rorp
else: return new_rp.getRORPath()
def patchonce_action(base_rp, basisrp, diff_rorp):
"""Return action patching basisrp using diff_rorp"""
assert diff_rorp, "Missing diff index %s" % basisrp.index
if not diff_rorp.lstat():
return robust.Action(None, lambda init_val: basisrp.delete(), None)
if Globals.preserve_hardlinks and diff_rorp.isflaglinked():
if not basisrp: basisrp = base_rp.new_index(diff_rorp.index)
tf = TempFile.new(basisrp)
def init(): Hardlink.link_rp(diff_rorp, tf, basisrp)
return robust.make_tf_robustaction(init, tf, basisrp)
elif basisrp and basisrp.isreg() and diff_rorp.isreg():
if diff_rorp.get_attached_filetype() != 'diff':
raise rpath.RPathException("File %s appears to have changed during"
" processing, skipping" % (basisrp.path,))
return Rdiff.patch_with_attribs_action(basisrp, diff_rorp)
else: # Diff contains whole file, just copy it over
if not basisrp: basisrp = base_rp.new_index(diff_rorp.index)
return robust.copy_with_attribs_action(diff_rorp, basisrp)
if (src_rorp and dest_rorp and src_rorp == dest_rorp and
(not Globals.preserve_hardlinks or
Hardlink.rorp_eq(src_rorp, dest_rorp))): continue
if statfileobj: statfileobj.add_changed(src_rorp, dest_rorp)
if not dest_rorp: yield src_rorp.index
else: yield dest_rorp.index
class IndexedTuple(UserList.UserList):
......@@ -277,12 +216,15 @@ def FillInIter(rpiter, rootrp):
(2,5). This is used when we need to process directories before or
after processing a file in that directory.
If start_index is given, start with start_index instead of ().
The indicies of rest of the rorps should also start with
start_index.
"""
# Handle first element as special case
first_rp = rpiter.next() # StopIteration gets passed upwards
cur_index = first_rp.index
for i in range(len(cur_index)):
yield rootrp.new_index(cur_index[:i])
for i in range(len(cur_index)): yield rootrp.new_index(cur_index[:i])
yield first_rp
del first_rp
old_index = cur_index
......@@ -294,7 +236,6 @@ def FillInIter(rpiter, rootrp):
for i in range(1, len(cur_index)): # i==0 case already handled
if cur_index[:i] != old_index[:i]:
filler_rp = rootrp.new_index(cur_index[:i])
assert filler_rp.isdir(), "This shouldn't be possible"
yield filler_rp
yield rp
old_index = cur_index
......@@ -318,6 +259,7 @@ class IterTreeReducer:
self.index = None
self.root_branch = branch_class(*branch_args)
self.branches = [self.root_branch]
self.root_fast_processed = None
def finish_branches(self, index):
"""Run Finish() on all branches index has passed
......@@ -355,6 +297,7 @@ class IterTreeReducer:
def Finish(self):
"""Call at end of sequence to tie everything up"""
if self.index is None or self.root_fast_processed: return
while 1:
to_be_finished = self.branches.pop()
to_be_finished.call_end_proc()
......@@ -375,7 +318,10 @@ class IterTreeReducer:
index = args[0]
if self.index is None:
self.root_branch.base_index = index
self.process_w_branch(self.root_branch, args)
if self.root_branch.can_fast_process(*args):
self.root_branch.fast_process(*args)
self.root_fast_processed = 1
else: self.process_w_branch(self.root_branch, args)
self.index = index
return 1
......@@ -389,7 +335,8 @@ class IterTreeReducer:
last_branch = self.branches[-1]
if last_branch.start_successful:
if last_branch.can_fast_process(*args):
last_branch.fast_process(*args)
robust.check_common_error(last_branch.on_error,
last_branch.fast_process, args)
else:
branch = self.add_branch(index)
self.process_w_branch(branch, args)
......@@ -452,7 +399,7 @@ class ITRBranch:
def log_prev_error(self, index):
"""Call function if no pending exception"""
log.Log("Skipping %s because of previous error" %
(os.path.join(*index),), 2)
log.Log("Skipping %s because of previous error" % \
(index and os.path.join(*index) or '()',), 2)
trunk/rdiff-backup/rdiff_backup/rpath.py
View file @ fe98b295
......@@ -35,7 +35,7 @@ are dealing with are local or remote.
"""
import os, stat, re, sys, shutil, gzip, socket, time, shutil
import os, stat, re, sys, shutil, gzip, socket, time
import Globals, FilenameMapping, Time, static, log
......@@ -81,7 +81,7 @@ def move(rpin, rpout):
copy(rpin, rpout)
rpin.delete()
def copy(rpin, rpout):
def copy(rpin, rpout, compress = 0):
"""Copy RPath rpin to rpout. Works for symlinks, dirs, etc."""
log.Log("Regular copying %s to %s" % (rpin.index, rpout.path), 6)
if not rpin.lstat():
......@@ -93,7 +93,7 @@ def copy(rpin, rpout):
rpout.delete() # easier to write that compare
else: return
if rpin.isreg(): copy_reg_file(rpin, rpout)
if rpin.isreg(): copy_reg_file(rpin, rpout, compress)
elif rpin.isdir(): rpout.mkdir()
elif rpin.issym(): rpout.symlink(rpin.readlink())
elif rpin.ischardev():
......@@ -106,15 +106,16 @@ def copy(rpin, rpout):
elif rpin.issock(): rpout.mksock()
else: raise RPathException("File %s has unknown type" % rpin.path)
def copy_reg_file(rpin, rpout):
def copy_reg_file(rpin, rpout, compress = 0):
"""Copy regular file rpin to rpout, possibly avoiding connection"""
try:
if rpout.conn is rpin.conn:
rpout.conn.shutil.copyfile(rpin.path, rpout.path)
if (rpout.conn is rpin.conn and
rpout.conn is not Globals.local_connection):
rpout.conn.rpath.copy_reg_file(rpin.path, rpout.path, compress)
rpout.setdata()
return
except AttributeError: pass
rpout.write_from_fileobj(rpin.open("rb"))
rpout.write_from_fileobj(rpin.open("rb"), compress = compress)
def cmp(rpin, rpout):
"""True if rpin has the same data as rpout
......@@ -179,9 +180,9 @@ def cmp_attribs(rp1, rp2):
(rp1.path, rp2.path, result), 7)
return result
def copy_with_attribs(rpin, rpout):
def copy_with_attribs(rpin, rpout, compress = 0):
"""Copy file and then copy over attributes"""
copy(rpin, rpout)
copy(rpin, rpout, compress)
if rpin.lstat(): copy_attribs(rpin, rpout)
def quick_cmp_with_attribs(rp1, rp2):
......@@ -278,9 +279,9 @@ class RORPath:
self.data[key] != other.data[key]): return None
return 1
def equal_verbose(self, other):
def equal_verbose(self, other, check_index = 1):
"""Like __eq__, but log more information. Useful when testing"""
if self.index != other.index:
if check_index and self.index != other.index:
log.Log("Index %s != index %s" % (self.index, other.index), 2)
return None
......@@ -372,6 +373,10 @@ class RORPath:
"""Return permission block of file"""
return self.data['perms']
def hassize(self):
"""True if rpath has a size parameter"""
return self.data.has_key('size')
def getsize(self):
"""Return length of file in bytes"""
return self.data['size']
......@@ -398,7 +403,8 @@ class RORPath:
def getnumlinks(self):
"""Number of places inode is linked to"""
return self.data['nlink']
try: return self.data['nlink']
except KeyError: return 1
def readlink(self):
"""Wrapper around os.readlink()"""
......@@ -446,9 +452,13 @@ class RORPath:
"""
return self.data.has_key('linked')
def flaglinked(self):
def get_link_flag(self):
"""Return previous index that a file is hard linked to"""
return self.data['linked']
def flaglinked(self, index):
"""Signal that rorp is a signature/diff for a hardlink file"""
self.data['linked'] = 1
self.data['linked'] = index
def open(self, mode):
"""Return file type object if any was given using self.setfile"""
......@@ -742,7 +752,6 @@ class RPath(RORPath):
def append_path(self, ext, new_index = ()):
"""Like append, but add ext to path instead of to index"""
assert not self.index # doesn't make sense if index isn't ()
return self.__class__(self.conn, "/".join((self.base, ext)), new_index)
def new_index(self, index):
......@@ -822,8 +831,8 @@ class RPath(RORPath):
return self.inc_type
def getinctime(self):
"""Return timestring of an increment file"""
return self.inc_timestr
"""Return time in seconds of an increment file"""
return Time.stringtotime(self.inc_timestr)
def getincbase(self):
"""Return the base filename of an increment file in rp form"""
......@@ -862,22 +871,4 @@ class RPathFileHook:
self.closing_thunk()
return result
# Import these late to avoid circular dependencies
#import FilenameMapping
#from lazy import *
#from selection import *
#from highlevel import *
#class RpathDeleter(ITRBranch):
# """Delete a directory. Called by RPath.delete()"""
# def start_process(self, index, rp):
# self.rp = rp
#
# def end_process(self):
# if self.rp.isdir(): self.rp.rmdir()
# else: self.rp.delete()
#
# def can_fast_process(self, index, rp): return not rp.isdir()
# def fast_process(self, index, rp): rp.delete()
trunk/rdiff-backup/rdiff_backup/selection.py
View file @ fe98b295
......@@ -94,29 +94,20 @@ class Select:
self.prefix = self.rpath.path
self.quoting_on = Globals.quoting_enabled and quoted_filenames
def set_iter(self, starting_index = None, iterate_parents = None,
sel_func = None):
def set_iter(self, iterate_parents = None, sel_func = None):
"""Initialize more variables, get ready to iterate
Will iterate indicies greater than starting_index. If
iterate_parents is true, will also include parents of
starting_index in iteration. Selection function sel_func is
called on each rpath and is usually self.Select. Returns self
just for convenience.
Selection function sel_func is called on each rpath and is
usually self.Select. Returns self just for convenience.
"""
if not sel_func: sel_func = self.Select
self.rpath.setdata() # this may have changed since Select init
if starting_index is not None:
self.starting_index = starting_index
self.iter = self.iterate_starting_from(self.rpath,
self.iterate_starting_from, sel_func)
elif self.quoting_on:
if self.quoting_on:
self.iter = self.Iterate(self.rpath, self.Iterate, sel_func)
else: self.iter = self.Iterate_fast(self.rpath, sel_func)
# only iterate parents if we are not starting from beginning
self.iterate_parents = starting_index is not None and iterate_parents
self.next = self.iter.next
self.__iter__ = lambda: self
return self
......@@ -149,6 +140,7 @@ class Select:
elif s == 2 and new_rpath.isdir(): yield (new_rpath, 1)
yield rpath
if not rpath.isdir(): return
diryield_stack = [diryield(rpath)]
delayed_rp_stack = []
......@@ -214,26 +206,6 @@ class Select:
for rp in rec_func(new_rp, rec_func, sel_func):
yield rp
def iterate_starting_from(self, rpath, rec_func, sel_func):
"""Like Iterate, but only yield indicies > self.starting_index"""
if rpath.index > self.starting_index: # past starting_index
for rp in self.Iterate(rpath, self.Iterate, sel_func):
yield rp
elif (rpath.index == self.starting_index[:len(rpath.index)]
and rpath.isdir()):
# May encounter starting index on this branch
if self.iterate_parents: yield rpath
for rp in self.iterate_in_dir(rpath, self.iterate_starting_from,
sel_func): yield rp
# def iterate_with_finalizer(self):
# """Like Iterate, but missing some options, and add finalizer"""
# finalize = IterTreeReducer(DestructiveSteppingFinalizer, ())
# for rp in self:
# yield rp
# finalize(rp.index, rp))
# finalize.Finish()
def Select(self, rp):
"""Run through the selection functions and return dominant val 0/1/2"""
for sf in self.selection_functions:
......
trunk/rdiff-backup/rdiff_backup/statistics.py
View file @ fe98b295
......@@ -19,8 +19,8 @@
"""Generate and process aggregated backup information"""
import re, os
import Globals, TempFile, robust, Time, rorpiter
import re, os, time
import Globals, TempFile, robust, Time, rorpiter, increment
class StatsException(Exception): pass
......@@ -73,6 +73,10 @@ class StatsObj:
"""Add 1 to value of attribute"""
self.__dict__[attr] += 1
def add_to_stat(self, attr, value):
"""Add value to given attribute"""
self.__dict__[attr] += value
def get_total_dest_size_change(self):
"""Return total destination size change
......@@ -215,12 +219,9 @@ class StatsObj:
def write_stats_to_rp(self, rp):
"""Write statistics string to given rpath"""
tf = TempFile.new(rp)
def init_thunk():
fp = tf.open("w")
fp.write(self.get_stats_string())
fp.close()
robust.make_tf_robustaction(init_thunk, (tf,), (rp,)).execute()
fp = rp.open("wb")
fp.write(self.get_stats_string())
assert not fp.close()
def read_stats_from_rp(self, rp):
"""Set statistics from rpath, return self for convenience"""
......@@ -263,81 +264,81 @@ class StatsObj:
return s
class ITRB(rorpiter.ITRBranch, StatsObj):
"""Keep track of per directory statistics
This is subclassed by the mirroring and incrementing ITRs.
"""
def __init__(self):
"""StatsITR initializer - zero out statistics"""
attr_dict = self.__dict__
for attr in StatsObj.stat_file_attrs: attr_dict[attr] = 0
self.ElapsedTime = self.Filename = None
def start_stats(self, mirror_dsrp):
"""Record status of mirror dsrp
class StatFileObj(StatsObj):
"""Build on StatsObj, add functions for processing files"""
def __init__(self, start_time = None):
"""StatFileObj initializer - zero out file attributes"""
StatsObj.__init__(self)
for attr in self.stat_file_attrs: self.set_stat(attr, 0)
if start_time is None: start_time = Time.curtime
self.StartTime = start_time
self.Errors = 0
This is called before the mirror is processed so we remember
the old state.
"""
if mirror_dsrp.lstat():
self.mirror_base_exists = 1
self.mirror_base_size = self.stats_getsize(mirror_dsrp)
else: self.mirror_base_exists = None
def stats_getsize(self, rp):
"""Return size of rp, with error checking"""
try: return rp.getsize()
except KeyError: return 0
def end_stats(self, diff_rorp, mirror_dsrp, inc_rp = None):
"""Set various statistics after mirror processed"""
if mirror_dsrp.lstat():
source_size = self.stats_getsize(mirror_dsrp)
self.SourceFiles += 1
self.SourceFileSize += source_size
if self.mirror_base_exists:
self.MirrorFiles += 1
self.MirrorFileSize += self.mirror_base_size
if diff_rorp: # otherwise no change
self.ChangedFiles += 1
self.ChangedSourceSize += source_size
self.ChangedMirrorSize += self.mirror_base_size
self.stats_incr_incfiles(inc_rp)
else: # new file was created
self.NewFiles += 1
self.NewFileSize += source_size
self.stats_incr_incfiles(inc_rp)
else:
if self.mirror_base_exists: # file was deleted from mirror
self.MirrorFiles += 1
self.MirrorFileSize += self.mirror_base_size
self.DeletedFiles += 1
self.DeletedFileSize += self.mirror_base_size
self.stats_incr_incfiles(inc_rp)
def fast_process(self, mirror_rorp):
"""Use when there is no change from source to mirror"""
source_size = self.stats_getsize(mirror_rorp)
def add_source_file(self, src_rorp):
"""Add stats of source file"""
self.SourceFiles += 1
self.MirrorFiles += 1
self.SourceFileSize += source_size
self.MirrorFileSize += source_size
def stats_incr_incfiles(self, inc_rp):
"""Increment IncrementFile statistics"""
if inc_rp:
self.IncrementFiles += 1
self.IncrementFileSize += self.stats_getsize(inc_rp)
def add_file_stats(self, branch):
"""Add all file statistics from branch to current totals"""
for attr in self.stat_file_attrs:
self.__dict__[attr] += branch.__dict__[attr]
if src_rorp.isreg(): self.SourceFileSize += src_rorp.getsize()
def add_dest_file(self, dest_rorp):
"""Add stats of destination size"""
self.MirrorFiles += 1
if dest_rorp.isreg(): self.MirrorFileSize += dest_rorp.getsize()
def add_changed(self, src_rorp, dest_rorp):
"""Update stats when src_rorp changes to dest_rorp"""
if src_rorp and src_rorp.lstat() and dest_rorp and dest_rorp.lstat():
self.ChangedFiles += 1
if src_rorp.isreg(): self.ChangedSourceSize += src_rorp.getsize()
if dest_rorp.isreg(): self.ChangedMirrorSize += dest_rorp.getsize()
elif src_rorp and src_rorp.lstat():
self.NewFiles += 1
if src_rorp.isreg(): self.NewFileSize += src_rorp.getsize()
elif dest_rorp and dest_rorp.lstat():
self.DeletedFiles += 1
if dest_rorp.isreg(): self.DeletedFileSize += dest_rorp.getsize()
def add_increment(self, inc_rorp):
"""Update stats with increment rorp"""
self.IncrementFiles += 1
if inc_rorp.isreg(): self.IncrementFileSize += inc_rorp.getsize()
def add_error(self):
"""Increment error stat by 1"""
self.Errors += 1
def finish(self, end_time = None):
"""Record end time and set other stats"""
if end_time is None: end_time = time.time()
self.EndTime = end_time
_active_statfileobj = None
def init_statfileobj():
"""Return new stat file object, record as active stat object"""
global _active_statfileobj
assert not _active_statfileobj, _active_statfileobj
_active_statfileobj = StatFileObj()
return _active_statfileobj
def get_active_statfileobj():
"""Return active stat file object if it exists"""
if _active_statfileobj: return _active_statfileobj
else: return None
def record_error():
"""Record error on active statfileobj, if there is one"""
if _active_statfileobj: _active_statfileobj.add_error()
def process_increment(inc_rorp):
"""Add statistics of increment rp incrp if there is active statfile"""
if _active_statfileobj: _active_statfileobj.add_increment(inc_rorp)
def write_active_statfileobj():
"""Write active StatFileObj object to session statistics file"""
global _active_statfileobj
assert _active_statfileobj
rp_base = Globals.rbdir.append("session_statistics")
session_stats_rp = increment.get_inc_ext(rp_base, 'data', Time.curtime)
_active_statfileobj.finish()
_active_statfileobj.write_stats_to_rp(session_stats_rp)
_active_statfileobj = None
trunk/rdiff-backup/testing/benchmark.py 0 → 100644
View file @ fe98b295
import sys, time
from commontest import *
from rdiff_backup import rpath, Globals
"""benchmark.py
When possible, use 'rdiff-backup' from the shell, which allows using
different versions of rdiff-backup by altering the PYTHONPATH. We
just use clock time, so this isn't exact at all.
"""
output_local = 1
output_desc = "testfiles/output"
new_pythonpath = None
def run_cmd(cmd):
"""Run the given cmd, return the amount of time it took"""
if new_pythonpath: full_cmd = "PYTHONPATH=%s %s" % (new_pythonpath, cmd)
else: full_cmd = cmd
print "Running command '%s'" % (full_cmd,)
t = time.time()
assert not os.system(full_cmd)
return time.time() - t
def create_many_files(dirname, s, count = 1000):
"""Create many short files in the dirname directory
There will be count files in the directory, and each file will
contain the string s.
"""
Myrm("testfiles/many_out")
dir_rp = rpath.RPath(Globals.local_connection, dirname)
dir_rp.mkdir()
for i in xrange(count):
rp = dir_rp.append(str(i))
fp = rp.open("wb")
fp.write(s)
assert not fp.close()
def create_nested(dirname, s, depth, branch_factor = 10):
"""Create many short files in branching directory"""
def write(rp):
fp = rp.open("wb")
fp.write(s)
assert not fp.close()
def helper(rp, depth):
rp.mkdir()
sub_rps = map(lambda i: rp.append(str(i)), range(branch_factor))
if depth == 1: map(write, sub_rps)
else: map(lambda rp: helper(rp, depth-1), sub_rps)
Myrm("testfiles/nested_out")
helper(rpath.RPath(Globals.local_connection, dirname), depth)
def benchmark(backup_cmd, restore_cmd, desc, update_func = None):
"""Print benchmark using backup_cmd and restore_cmd
If update_func is given, run it and then do backup a third time.
"""
print "Initially backing up %s: %ss" % (desc, run_cmd(backup_cmd))
print "Updating %s, no change: %ss" % (desc, run_cmd(backup_cmd))
if update_func:
update_func()
print "Updating %s, all changed: %ss" % (desc, run_cmd(backup_cmd))
Myrm("testfiles/rest_out")
print "Restoring %s to empty dir: %ss" % (desc, run_cmd(restore_cmd))
print "Restoring %s to unchanged dir: %ss" % (desc, run_cmd(restore_cmd))
def many_files():
"""Time backup and restore of 2000 files"""
count = 2000
create_many_files("testfiles/many_out", "a", count)
backup_cmd = "rdiff-backup testfiles/many_out " + output_desc
restore_cmd = "rdiff-backup --force -r now %s testfiles/rest_out" % \
(output_desc,)
update_func = lambda: create_many_files("testfiles/many_out", "e", count)
benchmark(backup_cmd, restore_cmd, "2000 1-byte files", update_func)
def many_files_rsync():
"""Test rsync benchmark"""
count = 2000
create_many_files("testfiles/many_out", "a", count)
rsync_command = ("rsync -e ssh -aH --delete testfiles/many_out " +
output_desc)
print "Initial rsync: %ss" % (run_cmd(rsync_command),)
print "rsync update: %ss" % (run_cmd(rsync_command),)
create_many_files("testfiles/many_out", "e", count)
print "Update changed rsync: %ss" % (run_cmd(rsync_command),)
def nested_files():
"""Time backup and restore of 10000 nested files"""
depth = 4
create_nested("testfiles/nested_out", "a", depth)
backup_cmd = "rdiff-backup testfiles/nested_out " + output_desc
restore_cmd = "rdiff-backup --force -r now %s testfiles/rest_out" % \
(output_desc,)
update_func = lambda: create_nested("testfiles/nested_out", "e", depth)
benchmark(backup_cmd, restore_cmd, "10000 1-byte nested files",
update_func)
def nested_files_rsync():
"""Test rsync on nested files"""
depth = 4
create_nested("testfiles/nested_out", "a", depth)
rsync_command = ("rsync -e ssh -aH --delete testfiles/nested_out " +
output_desc)
print "Initial rsync: %ss" % (run_cmd(rsync_command),)
print "rsync update: %ss" % (run_cmd(rsync_command),)
create_nested("testfiles/nested_out", "e", depth)
print "Update changed rsync: %ss" % (run_cmd(rsync_command),)
if len(sys.argv) < 2 or len(sys.argv) > 3:
print "Syntax: benchmark.py benchmark_func [output_description]"
print
print "Where output_description defaults to 'testfiles/output'."
print "Currently benchmark_func includes:"
print "'many_files', 'many_files_rsync', and, 'nested_files'."
sys.exit(1)
if len(sys.argv) == 3:
output_desc = sys.argv[2]
if ":" in output_desc: output_local = None
if output_local:
assert not rpath.RPath(Globals.local_connection, output_desc).lstat(), \
"Outfile file %s exists, try deleting it first" % (output_desc,)
if os.environ.has_key('BENCHMARKPYPATH'):
new_pythonpath = os.environ['BENCHMARKPYPATH']
function_name = sys.argv[1]
print "Running ", function_name
eval(sys.argv[1])()
trunk/rdiff-backup/testing/commontest.py
View file @ fe98b295
......@@ -3,7 +3,7 @@ import os, sys
from rdiff_backup.log import Log
from rdiff_backup.rpath import RPath
from rdiff_backup import Globals, Hardlink, SetConnections, Main, \
selection, highlevel, lazy, Time, rpath
selection, lazy, Time, rpath
SourceDir = "../src"
AbsCurdir = os.getcwd() # Absolute path name of current directory
......@@ -13,6 +13,9 @@ __no_execute__ = 1 # Keeps the actual rdiff-backup program from running
def Myrm(dirstring):
"""Run myrm on given directory string"""
root_rp = rpath.RPath(Globals.local_connection, dirstring)
for rp in selection.Select(root_rp).set_iter():
if rp.isdir(): rp.chmod(0700) # otherwise may not be able to remove
assert not os.system("rm -rf %s" % (dirstring,))
def Make():
......@@ -21,6 +24,13 @@ def Make():
os.system("python ./Make")
os.chdir(AbsCurdir)
def MakeOutputDir():
"""Initialize the testfiles/output directory"""
Myrm("testfiles/output")
rp = rpath.RPath(Globals.local_connection, "testfiles/output")
rp.mkdir()
return rp
def rdiff_backup(source_local, dest_local, src_dir, dest_dir,
current_time = None, extra_options = ""):
"""Run rdiff-backup with the given options
......@@ -121,6 +131,7 @@ def InternalRestore(mirror_local, dest_local, mirror_dir, dest_dir, time):
the testing directory and will be modified for remote trials.
"""
Main.force = 1
remote_schema = '%s'
#_reset_connections()
if not mirror_local:
......
trunk/rdiff-backup/testing/finaltest.py
View file @ fe98b295
import unittest, os, re, sys, time
from commontest import *
from rdiff_backup import Globals, log, rpath
from rdiff_backup import Globals, log, rpath, robust
"""Regression tests"""
Globals.exclude_mirror_regexps = [re.compile(".*/rdiff-backup-data")]
log.Log.setverbosity(7)
log.Log.setverbosity(3)
lc = Globals.local_connection
......@@ -43,7 +43,7 @@ class PathSetter(unittest.TestCase):
def reset_schema(self):
self.rb_schema = SourceDir + \
"/../rdiff-backup -v3 --remote-schema './chdir-wrapper2 %s' "
"/../rdiff-backup -v7 --remote-schema './chdir-wrapper2 %s' "
def refresh(self, *rp_list):
"""Reread data for the given rps"""
......@@ -168,8 +168,9 @@ class PathSetter(unittest.TestCase):
def getinc_paths(self, basename, directory):
"""Return increment.______.dir paths"""
incfiles = filter(lambda s: s.startswith(basename),
os.listdir(directory))
dirrp = rpath.RPath(Globals.local_connection, directory)
incfiles = [filename for filename in robust.listrp(dirrp)
if filename.startswith(basename)]
incfiles.sort()
incrps = map(lambda f: rpath.RPath(lc, directory+"/"+f), incfiles)
return map(lambda x: x.path, filter(rpath.RPath.isincfile, incrps))
......@@ -196,26 +197,15 @@ class Final(PathSetter):
self.set_connections(None, None, "test2/tmp", "../../")
self.runtest()
# def testMirroringLocal(self):
# """Run mirroring only everything remote"""
# self.delete_tmpdirs()
# self.set_connections(None, None, None, None)
# self.exec_rb_extra_args(10000, "-m",
# "testfiles/various_file_types",
# "testfiles/output")
# assert CompareRecursive(Local.vftrp, Local.rpout, exclude_rbdir = None)
# def testMirroringRemote(self):
# """Run mirroring only everything remote"""
# self.delete_tmpdirs()
# self.set_connections("test1/", "../", "test2/tmp/", "../../")
# self.exec_rb_extra_args(10000, "-m",
# "testfiles/various_file_types",
# "testfiles/output")
# assert CompareRecursive(Local.vftrp, Local.rpout, exclude_rbdir = None)
def testProcLocal(self):
"""Test initial backup of /proc locally"""
Myrm("testfiles/procoutput")
self.set_connections(None, None, None, None)
self.exec_rb(None, '../../../../../../proc', 'testfiles/procoutput')
def testProcRemote(self):
"""Test mirroring proc"""
Myrm("testfiles/procoutput")
self.set_connections(None, None, "test2/tmp/", "../../")
self.exec_rb(None, '../../../../../../proc', 'testfiles/procoutput')
......@@ -352,44 +342,5 @@ testfiles/increment2/changed_dir""")
self.assertRaises(OSError, os.lstat,
'testfiles/restoretarget1/executable2')
class FinalCorrupt(PathSetter):
def testBackupOverlay(self):
"""Test backing up onto a directory already backed up for that time
This will test to see if rdiff-backup will ignore files who
already have an increment where it wants to put something.
Just make sure rdiff-backup doesn't exit with an error.
"""
self.delete_tmpdirs()
assert not os.system("cp -a testfiles/corruptbackup testfiles/output")
self.set_connections(None, None, None, None)
self.exec_rb(None, 'testfiles/corruptbackup_source',
'testfiles/output')
def testBackupOverlayRemote(self):
"""Like above but destination is remote"""
self.delete_tmpdirs()
assert not os.system("cp -a testfiles/corruptbackup testfiles/output")
self.set_connections(None, None, "test1/", '../')
self.exec_rb(None, 'testfiles/corruptbackup_source',
'testfiles/output')
def testCheckpointData(self):
"""Destination directory has bad checkpoint data, no sym"""
self.delete_tmpdirs()
assert not os.system("cp -a testfiles/corrupt_dest1 testfiles/output")
self.set_connections(None, None, None, None)
self.exec_rb(None, 'testfiles/various_file_types', 'testfiles/output')
def testCheckpointData2(self):
"""Destination directory has bad checkpoint data, with sym"""
self.delete_tmpdirs()
assert not os.system("cp -a testfiles/corrupt_dest2 testfiles/output")
self.set_connections(None, None, None, None)
self.exec_rb(None, 'testfiles/various_file_types', 'testfiles/output')
if __name__ == "__main__": unittest.main()
trunk/rdiff-backup/testing/hardlinktest.py
View file @ fe98b295
import os, unittest
import os, unittest, time
from commontest import *
from rdiff_backup import Globals, Hardlink, selection, rpath
......@@ -30,48 +29,6 @@ class HardlinkTest(unittest.TestCase):
assert not CompareRecursive(self.hardlink_dir1, self.hardlink_dir2,
compare_hardlinks = 1)
def testCheckpointing(self):
"""Test saving and recovering of various dictionaries"""
d1 = {1:1}
d2 = {2:2}
d3 = {3:3}
d4 = {}
Hardlink._src_inode_indicies = d1
Hardlink._src_index_indicies = d2
Hardlink._dest_inode_indicies = d3
Hardlink._dest_index_indicies = d4
self.reset_output()
Time.setcurtime(12345)
Globals.isbackup_writer = 1
Hardlink.final_checkpoint(self.outputrp)
reset_hardlink_dicts()
assert Hardlink.retrieve_checkpoint(self.outputrp, 12345)
assert Hardlink._src_inode_indicies == d1, \
Hardlink._src_inode_indicies
assert Hardlink._src_index_indicies == d2, \
Hardlink._src_index_indicies
assert Hardlink._dest_inode_indicies == d3, \
Hardlink._dest_inode_indicies
assert Hardlink._dest_index_indicies == d4, \
Hardlink._dest_index_indicies
def testFinalwrite(self):
"""Test writing of the final database"""
Globals.isbackup_writer = 1
Time.setcurtime(123456)
Globals.rbdir = self.outputrp
finald = Hardlink._src_index_indicies = {'hello':'world'}
self.reset_output()
Hardlink.final_writedata()
Hardlink._src_index_indicies = None
assert Hardlink.retrieve_final(123456)
assert Hardlink._src_index_indicies == finald
def testBuildingDict(self):
"""See if the partial inode dictionary is correct"""
Globals.preserve_hardlinks = 1
......@@ -143,6 +100,74 @@ class HardlinkTest(unittest.TestCase):
BackupRestoreSeries(None, None, dirlist, compare_hardlinks=1)
BackupRestoreSeries(1, 1, dirlist, compare_hardlinks=1)
def testInnerRestore(self):
"""Restore part of a dir, see if hard links preserved"""
MakeOutputDir()
output = rpath.RPath(Globals.local_connection,
"testfiles/output")
# Now set up directories out_hardlink1 and out_hardlink2
hlout1 = rpath.RPath(Globals.local_connection,
"testfiles/out_hardlink1")
if hlout1.lstat(): hlout1.delete()
hlout1.mkdir()
hlout1_sub = hlout1.append("subdir")
hlout1_sub.mkdir()
hl1_1 = hlout1_sub.append("hardlink1")
hl1_2 = hlout1_sub.append("hardlink2")
hl1_3 = hlout1_sub.append("hardlink3")
hl1_4 = hlout1_sub.append("hardlink4")
# 1 and 2 are hard linked, as are 3 and 4
hl1_1.touch()
hl1_2.hardlink(hl1_1.path)
hl1_3.touch()
hl1_4.hardlink(hl1_3.path)
hlout2 = rpath.RPath(Globals.local_connection,
"testfiles/out_hardlink2")
if hlout2.lstat(): hlout2.delete()
assert not os.system("cp -a testfiles/out_hardlink1 "
"testfiles/out_hardlink2")
hlout2_sub = hlout2.append("subdir")
hl2_1 = hlout2_sub.append("hardlink1")
hl2_2 = hlout2_sub.append("hardlink2")
hl2_3 = hlout2_sub.append("hardlink3")
hl2_4 = hlout2_sub.append("hardlink4")
# Now 2 and 3 are hard linked, also 1 and 4
rpath.copy_with_attribs(hl1_1, hl2_1)
rpath.copy_with_attribs(hl1_2, hl2_2)
hl2_3.delete()
hl2_3.hardlink(hl2_2.path)
hl2_4.delete()
hl2_4.hardlink(hl2_1.path)
rpath.copy_attribs(hlout1_sub, hlout2_sub)
InternalBackup(1, 1, hlout1.path, output.path)
time.sleep(1)
InternalBackup(1, 1, hlout2.path, output.path)
out2 = rpath.RPath(Globals.local_connection, "testfiles/out2")
hlout1 = out2.append("hardlink1")
hlout2 = out2.append("hardlink2")
hlout3 = out2.append("hardlink3")
hlout4 = out2.append("hardlink4")
if out2.lstat(): out2.delete()
InternalRestore(1, 1, "testfiles/output/subdir", "testfiles/out2", 1)
out2.setdata()
for rp in [hlout1, hlout2, hlout3, hlout4]: rp.setdata()
assert hlout1.getinode() == hlout2.getinode()
assert hlout3.getinode() == hlout4.getinode()
assert hlout1.getinode() != hlout3.getinode()
if out2.lstat(): out2.delete()
InternalRestore(1, 1, "testfiles/output/subdir", "testfiles/out2",
int(time.time()))
out2.setdata()
for rp in [hlout1, hlout2, hlout3, hlout4]: rp.setdata()
assert hlout1.getinode() == hlout4.getinode()
assert hlout2.getinode() == hlout3.getinode()
assert hlout1.getinode() != hlout2.getinode()
if __name__ == "__main__": unittest.main()
trunk/rdiff-backup/testing/incrementtest.py
View file @ fe98b295
import unittest, os, re, time
from commontest import *
from rdiff_backup import log, rpath, restore, increment, Time, \
Rdiff, statistics
from rdiff_backup import log, rpath, increment, Time, Rdiff, statistics
lc = Globals.local_connection
Globals.change_source_perms = 1
......@@ -21,14 +20,14 @@ dir = getrp(".")
sym = getrp("symbolic_link")
nothing = getrp("nothing")
target = rpath.RPath(lc, "testfiles/out")
out2 = rpath.RPath(lc, "testfiles/out2")
out_gz = rpath.RPath(lc, "testfiles/out.gz")
target = rpath.RPath(lc, "testfiles/output/out")
out2 = rpath.RPath(lc, "testfiles/output/out2")
out_gz = rpath.RPath(lc, "testfiles/output/out.gz")
Time.setprevtime(999424113)
prevtimestr = "2001-09-02T02:48:33-07:00"
t_pref = "testfiles/out.2001-09-02T02:48:33-07:00"
t_diff = "testfiles/out.2001-09-02T02:48:33-07:00.diff"
t_pref = "testfiles/output/out.2001-09-02T02:48:33-07:00"
t_diff = "testfiles/output/out.2001-09-02T02:48:33-07:00.diff"
Globals.no_compression_regexp = \
re.compile(Globals.no_compression_regexp_string, re.I)
......@@ -37,11 +36,12 @@ class inctest(unittest.TestCase):
"""Test the incrementRP function"""
def setUp(self):
Globals.set('isbackup_writer',1)
MakeOutputDir()
def check_time(self, rp):
"""Make sure that rp is an inc file, and time is Time.prevtime"""
assert rp.isincfile(), rp
t = Time.stringtotime(rp.getinctime())
t = rp.getinctime()
assert t == Time.prevtime, (t, Time.prevtime)
def testreg(self):
......@@ -114,7 +114,7 @@ class inctest(unittest.TestCase):
rp = increment.Increment(rf, rf2, target)
self.check_time(rp)
assert rpath.cmp_attribs(rp, rf2)
Rdiff.patch_action(rf, rp, out2).execute()
Rdiff.patch_local(rf, rp, out2)
assert rpath.cmp(rf2, out2)
rp.delete()
out2.delete()
......@@ -125,7 +125,7 @@ class inctest(unittest.TestCase):
rp = increment.Increment(rf, rf2, target)
self.check_time(rp)
assert rpath.cmp_attribs(rp, rf2)
Rdiff.patch_action(rf, rp, out2, delta_compressed = 1).execute()
Rdiff.patch_local(rf, rp, out2, delta_compressed = 1)
assert rpath.cmp(rf2, out2)
rp.delete()
out2.delete()
......@@ -139,86 +139,10 @@ class inctest(unittest.TestCase):
rp = increment.Increment(rf, out_gz, target)
self.check_time(rp)
assert rpath.cmp_attribs(rp, out_gz)
Rdiff.patch_action(rf, rp, out2).execute()
Rdiff.patch_local(rf, rp, out2)
assert rpath.cmp(out_gz, out2)
rp.delete()
out2.delete()
out_gz.delete()
class inctest2(unittest.TestCase):
"""Like inctest but contains more elaborate tests"""
def stats_check_initial(self, s):
"""Make sure stats object s compatible with initial mirroring
A lot of the off by one stuff is because the root directory
exists in the below examples.
"""
assert s.MirrorFiles == 1 or s.MirrorFiles == 0
assert s.MirrorFileSize < 20000
assert s.NewFiles <= s.SourceFiles <= s.NewFiles + 1
assert s.NewFileSize <= s.SourceFileSize <= s.NewFileSize + 20000
assert s.ChangedFiles == 1 or s.ChangedFiles == 0
assert s.ChangedSourceSize < 20000
assert s.ChangedMirrorSize < 20000
assert s.DeletedFiles == s.DeletedFileSize == 0
assert s.IncrementFileSize == 0
def testStatistics(self):
"""Test the writing of statistics
The file sizes are approximate because the size of directories
could change with different file systems...
"""
Globals.compression = 1
Myrm("testfiles/output")
InternalBackup(1, 1, "testfiles/stattest1", "testfiles/output")
InternalBackup(1, 1, "testfiles/stattest2", "testfiles/output",
time.time()+1)
rbdir = rpath.RPath(Globals.local_connection,
"testfiles/output/rdiff-backup-data")
#incs = Restore.get_inclist(rbdir.append("subdir").
# append("directory_statistics"))
#assert len(incs) == 2
#s1 = StatsObj().read_stats_from_rp(incs[0]) # initial mirror stats
#assert s1.SourceFiles == 2
#assert 400000 < s1.SourceFileSize < 420000
#self.stats_check_initial(s1)
#subdir_stats = StatsObj().read_stats_from_rp(incs[1]) # increment stats
#assert subdir_stats.SourceFiles == 2
#assert 400000 < subdir_stats.SourceFileSize < 420000
#assert subdir_stats.MirrorFiles == 2
#assert 400000 < subdir_stats.MirrorFileSize < 420000
#assert subdir_stats.NewFiles == subdir_stats.NewFileSize == 0
#assert subdir_stats.DeletedFiles == subdir_stats.DeletedFileSize == 0
#assert subdir_stats.ChangedFiles == 2
#assert 400000 < subdir_stats.ChangedSourceSize < 420000
#assert 400000 < subdir_stats.ChangedMirrorSize < 420000
#assert 10 < subdir_stats.IncrementFileSize < 20000
incs = restore.get_inclist(rbdir.append("session_statistics"))
assert len(incs) == 2
s2 = statistics.StatsObj().read_stats_from_rp(incs[0])
assert s2.SourceFiles == 7
assert 700000 < s2.SourceFileSize < 750000
self.stats_check_initial(s2)
root_stats = statistics.StatsObj().read_stats_from_rp(incs[1])
assert root_stats.SourceFiles == 7, root_stats.SourceFiles
assert 550000 < root_stats.SourceFileSize < 570000
assert root_stats.MirrorFiles == 7
assert 700000 < root_stats.MirrorFileSize < 750000
assert root_stats.NewFiles == 1
assert root_stats.NewFileSize == 0
assert root_stats.DeletedFiles == 1
assert root_stats.DeletedFileSize == 200000
assert 3 <= root_stats.ChangedFiles <= 4, root_stats.ChangedFiles
assert 450000 < root_stats.ChangedSourceSize < 470000
assert 400000 < root_stats.ChangedMirrorSize < 420000
assert 10 < root_stats.IncrementFileSize < 30000
if __name__ == '__main__': unittest.main()
trunk/rdiff-backup/testing/metadatatest.py
View file @ fe98b295
import unittest, os, cStringIO, time
from rdiff_backup.metadata import *
from rdiff_backup import rpath, connection, Globals, selection, \
destructive_stepping
from rdiff_backup import rpath, connection, Globals, selection
tempdir = rpath.RPath(Globals.local_connection, "testfiles/output")
......
trunk/rdiff-backup/testing/rdifftest.py
View file @ fe98b295
......@@ -50,7 +50,7 @@ class RdiffTest(unittest.TestCase):
self.delta.write_from_fileobj(Rdiff.get_delta_sigrp(self.signature,
self.new))
assert self.delta.lstat()
Rdiff.patch_action(self.basis, self.delta, self.output).execute()
Rdiff.patch_local(self.basis, self.delta, self.output)
assert rpath.cmp(self.new, self.output)
map(rpath.RPath.delete, rplist)
......@@ -74,14 +74,14 @@ class RdiffTest(unittest.TestCase):
os.system("mv %s %s" % (self.delta.path + ".gz", self.delta.path))
self.delta.setdata()
Rdiff.patch_action(self.basis, self.delta, self.output,
delta_compressed = 1).execute()
Rdiff.patch_local(self.basis, self.delta, self.output,
delta_compressed = 1)
assert rpath.cmp(self.new, self.output)
map(rpath.RPath.delete, rplist)
def testWriteDelta(self):
"""Test write delta feature of rdiff"""
self.delta.delete()
if self.delta.lstat(): self.delta.delete()
rplist = [self.basis, self.new, self.delta, self.output]
MakeRandomFile(self.basis.path)
MakeRandomFile(self.new.path)
......@@ -90,7 +90,7 @@ class RdiffTest(unittest.TestCase):
Rdiff.write_delta(self.basis, self.new, self.delta)
assert self.delta.lstat()
Rdiff.patch_action(self.basis, self.delta, self.output).execute()
Rdiff.patch_local(self.basis, self.delta, self.output)
assert rpath.cmp(self.new, self.output)
map(rpath.RPath.delete, rplist)
......@@ -109,7 +109,7 @@ class RdiffTest(unittest.TestCase):
os.system("gunzip " + delta_gz.path)
delta_gz.setdata()
self.delta.setdata()
Rdiff.patch_action(self.basis, self.delta, self.output).execute()
Rdiff.patch_local(self.basis, self.delta, self.output)
assert rpath.cmp(self.new, self.output)
map(rpath.RPath.delete, rplist)
......@@ -128,7 +128,7 @@ class RdiffTest(unittest.TestCase):
self.delta.write_from_fileobj(Rdiff.get_delta_sigrp(self.signature,
self.new))
assert self.delta.lstat()
Rdiff.patch_action(self.basis, self.delta).execute()
Rdiff.patch_local(self.basis, self.delta)
assert rpath.cmp(self.basis, self.new)
map(rpath.RPath.delete, rplist)
......@@ -141,31 +141,10 @@ class RdiffTest(unittest.TestCase):
MakeRandomFile(self.basis.path)
MakeRandomFile(self.new.path)
map(rpath.RPath.setdata, rplist)
Rdiff.copy_action(self.basis, self.new).execute()
Rdiff.copy_local(self.basis, self.new)
assert rpath.cmp(self.basis, self.new)
map(rpath.RPath.delete, rplist)
def testPatchWithAttribs(self):
"""Using rdiff to copy two files with attributes"""
rplist = [self.basis, self.new, self.delta]
for rp in rplist:
if rp.lstat(): rp.delete()
MakeRandomFile(self.basis.path)
MakeRandomFile(self.new.path)
self.new.chmod(0401)
map(rpath.RPath.setdata, rplist)
Rdiff.write_delta(self.basis, self.new, self.delta)
rpath.copy_attribs(self.new, self.delta)
assert self.delta.getperms() == 0401
assert not self.basis == self.new
Rdiff.patch_with_attribs_action(self.basis, self.delta).execute()
if not self.basis == self.new:
print self.basis, self.new
assert 0
map(rpath.RPath.delete, rplist)
if __name__ == '__main__':
unittest.main()
trunk/rdiff-backup/testing/regressiontest.py
View file @ fe98b295
import unittest, os
from commontest import *
from rdiff_backup import Globals, SetConnections, log, rpath
from rdiff_backup import Globals, SetConnections, log, rpath, backup
"""Regression tests
......@@ -12,14 +12,14 @@ testfiles
Globals.set('change_source_perms', 1)
Globals.counter = 0
log.Log.setverbosity(7)
log.Log.setverbosity(3)
def get_local_rp(extension):
return rpath.RPath(Globals.local_connection, "testfiles/" + extension)
class Local:
"""This is just a place to put increments relative to the local
connection"""
def get_local_rp(extension):
return rpath.RPath(Globals.local_connection, "testfiles/" + extension)
inc1rp = get_local_rp('increment1')
inc2rp = get_local_rp('increment2')
inc3rp = get_local_rp('increment3')
......@@ -152,98 +152,52 @@ class IncrementTest1(unittest.TestCase):
"""Increment/Restore when both directories are remote"""
BackupRestoreSeries(None, None, self.dirlist)
def testNoWrite(self):
"""Test backup/restore on dirs without write permissions"""
def write_string(rp, s = ""):
"""Write string s to file"""
fp = rp.open("wb")
fp.write(s)
assert not fp.close()
class IncrementTest2(PathSetter):
def OldtestRecoveryLocal(self):
"""Test to see if rdiff-backup can continue with bad increment"""
assert not os.system("rm -rf testfiles/recovery_out_backup")
self.setPathnames(None, None, None, None)
Time.setprevtime(1006136450)
Time.setcurtime()
Globals.add_regexp('.*rdiff-backup-data', 1)
os.system('cp -a testfiles/recovery_out testfiles/recovery_out_backup')
recovery_in = self.get_src_rp('testfiles/recovery')
recovery_out = self.get_dest_rp('testfiles/recovery_out_backup')
recovery_inc = self.get_dest_rp('testfiles/recovery_out_backup/'
'rdiff-backup-data/increments')
highlevel.Mirror_and_increment(recovery_in, recovery_out, recovery_inc)
# Should probably check integrity of increments, but for now
# allow if it doesn't during the Mirror_and_increment
def OldtestRecoveryRemote(self):
"""Test Recovery with both connections remote"""
assert not os.system('rm -rf testfiles/recovery_out_backup')
self.setPathnames('test1', '../', 'test2/tmp', '../../')
Time.setprevtime(1006136450)
Time.setcurtime()
Globals.add_regexp('.*rdiff-backup-data', 1)
os.system('cp -a testfiles/recovery_out testfiles/recovery_out_backup')
recovery_in = self.get_src_rp('testfiles/recovery')
recovery_out = self.get_dest_rp('testfiles/recovery_out_backup')
recovery_inc = self.get_dest_rp('testfiles/recovery_out_backup/'
'rdiff-backup-data/increments')
highlevel.Mirror_and_increment(recovery_in, recovery_out, recovery_inc)
# Should probably check integrity of increments, but for now
# allow if it doesn't during the Mirror_and_increment
def runtest(self):
"""After setting connections, etc., run actual test using this"""
Time.setcurtime()
Main.backup_set_select(Local.inc1rp)
highlevel.Mirror(self.inc1rp, self.rpout)
assert CompareRecursive(Local.inc1rp, Local.rpout)
Time.setcurtime()
Time.setprevtime(999500000)
Main.backup_set_select(self.inc2rp)
highlevel.Mirror_and_increment(self.inc2rp, self.rpout, self.rpout_inc)
assert CompareRecursive(Local.inc2rp, Local.rpout)
def make_subdirs():
"""Make testfiles/no_write_out and testfiles/no_write_out2"""
nw_out1 = get_local_rp("no_write_out")
nw_out1.mkdir()
Time.setcurtime()
Time.setprevtime(999510000)
Main.backup_set_select(self.inc3rp)
highlevel.Mirror_and_increment(self.inc3rp, self.rpout, self.rpout_inc)
assert CompareRecursive(Local.inc3rp, Local.rpout)
nw_out1_1 = get_local_rp("no_write_out/1")
write_string(nw_out1_1)
nw_out1_1.chmod(0)
Time.setcurtime()
Time.setprevtime(999520000)
Main.backup_set_select(self.inc4rp)
highlevel.Mirror_and_increment(self.inc4rp, self.rpout, self.rpout_inc)
assert CompareRecursive(Local.inc4rp, Local.rpout)
nw_out1_2 = get_local_rp("no_write_out/2")
write_string(nw_out1_2, 'e')
nw_out1_1.chmod(0400)
print "Restoring to self.inc4"
highlevel.Restore(999530000, self.rpout, self.get_inctup(),
self.rpout4)
assert CompareRecursive(Local.inc4rp, Local.rpout4)
nw1_sub = get_local_rp("no_write_out/subdir")
nw1_sub.mkdir()
print "Restoring to self.inc3"
highlevel.Restore(999520000, self.rpout, self.get_inctup(),
self.rpout3)
assert CompareRecursive(Local.inc3rp, Local.rpout3)
nw_out1_sub1 = get_local_rp("no_write_out/subdir/1")
write_string(nw_out1_sub1, 'f')
nw1_sub.chmod(0500)
nw_out1.chmod(0500)
print "Restoring to self.inc2"
highlevel.Restore(999510000, self.rpout, self.get_inctup(),
self.rpout2)
assert CompareRecursive(Local.inc2rp, Local.rpout2)
nw_out2 = get_local_rp("no_write_out2")
nw_out2.mkdir()
print "Restoring to self.inc1"
highlevel.Restore(999500000, self.rpout, self.get_inctup(),
self.rpout1)
assert CompareRecursive(Local.inc1rp, Local.rpout1)
nw_out2_1 = get_local_rp("no_write_out2/1")
write_string(nw_out2_1, 'g')
def get_inctup(self):
"""Return inc tuples as expected by Restore.RestoreRecursive
nw_out2_2 = get_local_rp("no_write_out2/2")
write_string(nw_out2_2, 'aeu')
nw_out1.chmod(0500)
Assumes output increment directory is
testfiles/output_inc._____.
"""
filenames = filter(lambda x: x.startswith("output_inc."),
Local.prefix.listdir())
rplist = map(lambda x: Local.prefix.append(x), filenames)
return IndexedTuple((), (Local.prefix.append("output_inc"), rplist))
Myrm("testfiles/no_write_out")
Myrm("testfiles/no_write_out2")
Myrm("testfiles/output")
make_subdirs()
BackupRestoreSeries(1, 1, ['testfiles/no_write_out',
'testfiles/no_write_out2',
'testfiles/empty'])
class MirrorTest(PathSetter):
......@@ -317,7 +271,7 @@ class MirrorTest(PathSetter):
Globals.change_ownership = 1
self.refresh(self.rootfiles, self.rootfiles_out,
Local.rootfiles, Local.rootfiles_out) # add uid/gid info
highlevel.Mirror(self.rootfiles, self.rootfiles_out)
backup.Mirror(self.rootfiles, self.rootfiles_out)
assert CompareRecursive(Local.rootfiles, Local.rootfiles_out)
Globals.change_ownership = None
self.refresh(self.rootfiles, self.rootfiles_out,
......@@ -330,29 +284,13 @@ class MirrorTest(PathSetter):
conn.Globals.set('change_ownership', 1)
self.refresh(self.rootfiles, self.rootfiles_out,
Local.rootfiles, Local.rootfiles_out) # add uid/gid info
highlevel.Mirror(self.rootfiles, self.rootfiles_out)
backup.Mirror(self.rootfiles, self.rootfiles_out)
assert CompareRecursive(Local.rootfiles, Local.rootfiles_out)
for coon in Globals.connections:
conn.Globals.set('change_ownership', None)
self.refresh(self.rootfiles, self.rootfiles_out,
Local.rootfiles, Local.rootfiles_out) # remove that info
def testRoot2Local(self):
"""Make sure we can backup a directory we don't own"""
self.setPathnames(None, None, None, None)
Globals.change_ownership = Globals.change_source_perms = None
self.refresh(self.rootfiles2, self.rootfiles_out2,
Local.rootfiles2, Local.rootfiles_out2) # add uid/gid info
self.Mirror(self.rootfiles2, self.rootfiles_out2)
assert CompareRecursive(Local.rootfiles2, Local.rootfiles_out2)
self.refresh(self.rootfiles2, self.rootfiles_out2,
Local.rootfiles2, Local.rootfiles_out2) # remove that info
self.Mirror(self.rootfiles21, self.rootfiles_out2)
assert CompareRecursive(Local.rootfiles21, Local.rootfiles_out2)
self.refresh(self.rootfiles21, self.rootfiles_out2,
Local.rootfiles21, Local.rootfiles_out2) # remove that info
Globals.change_source_perms = 1
def deleteoutput(self):
assert not os.system("rm -rf testfiles/output*")
self.rbdir = self.rpout.append('rdiff-backup-data')
......@@ -395,12 +333,12 @@ class MirrorTest(PathSetter):
assert CompareRecursive(Local.inc2rp, Local.rpout)
def Mirror(self, rpin, rpout):
"""Like highlevel.Mirror, but run misc_setup first"""
"""Like backup.Mirror, but setup first, cleanup later"""
Main.force = 1
Main.misc_setup([rpin, rpout])
Main.backup_set_select(rpin)
Main.backup_init_dirs(rpin, rpout)
highlevel.Mirror(rpin, rpout)
backup.Mirror(rpin, rpout)
Log.close_logfile()
Hardlink.clear_dictionaries()
......
trunk/rdiff-backup/testing/restoretest.py
View file @ fe98b295
import unittest
from commontest import *
from rdiff_backup import log, restore, Globals, rpath
from rdiff_backup import log, restore, Globals, rpath, TempFile
Log.setverbosity(3)
lc = Globals.local_connection
tempdir = rpath.RPath(Globals.local_connection, "testfiles/output")
restore_base_rp = rpath.RPath(Globals.local_connection,
"testfiles/restoretest")
restore_base_filenames = restore_base_rp.listdir()
mirror_time = 1041109438 # just some late time
class RestoreFileComparer:
"""Holds a file to be restored and tests against it
Each object has a restore file and a dictionary of times ->
rpaths. When the restore file is restored to one of the given
times, the resulting file should be the same as the related rpath.
"""
def __init__(self, rf):
self.rf = rf
self.time_rp_dict = {}
def add_rpath(self, rp, t):
"""Add rp, which represents what rf should be at given time t"""
assert not self.time_rp_dict.has_key(t)
self.time_rp_dict[t] = rp
def compare_at_time(self, t):
"""Restore file, make sure it is the same at time t"""
log.Log("Checking result at time %s" % (t,), 7)
tf = TempFile.new(tempdir.append("foo"))
restore._mirror_time = mirror_time
restore._rest_time = t
self.rf.set_relevant_incs()
out_rorpath = self.rf.get_attribs().getRORPath()
correct_result = self.time_rp_dict[t]
if out_rorpath.isreg():
out_rorpath.setfile(self.rf.get_restore_fp())
rpath.copy_with_attribs(out_rorpath, tf)
assert tf.equal_verbose(correct_result, check_index = 0), \
"%s, %s" % (tf, correct_result)
if tf.isreg():
assert rpath.cmpfileobj(tf.open("rb"), correct_result.open("rb"))
if tf.lstat(): tf.delete()
def compare_all(self):
"""Check restore results for all available times"""
for t in self.time_rp_dict.keys(): self.compare_at_time(t)
class RestoreTest(unittest.TestCase):
"""Test Restore class"""
prefix = "testfiles/restoretest/"
def maketesttuples(self, basename):
"""Make testing tuples from available files starting with prefix
tuples is a sorted (oldest to newest) list of pairs (rp1, rp2)
where rp1 is an increment file and rp2 is the same but without
the final extension. incs is a list of all increment files.
"""
dirlist = os.listdir(self.prefix)
dirlist.sort()
baselist = filter(lambda f: f.startswith(basename), dirlist)
rps = map(lambda f: rpath.RPath(lc, self.prefix+f), baselist)
incs = filter(lambda rp: rp.isincfile(), rps)
tuples = map(lambda rp: (rp, rpath.RPath(lc, "%s.%s" %
(rp.getincbase().path,
rp.getinctime()))),
incs)
return tuples, incs
def restoreonefiletest(self, basename):
tuples, incs = self.maketesttuples(basename)
rpbase = rpath.RPath(lc, self.prefix + basename)
rptarget = rpath.RPath(lc, "testfiles/outfile")
Hardlink.initialize_dictionaries()
for pair in tuples:
print "Processing file " + pair[0].path
if rptarget.lstat(): rptarget.delete()
rest_time = Time.stringtotime(pair[0].getinctime())
rid = restore.RestoreIncrementData((), rpbase, incs)
rid.sortincseq(rest_time, 10000000000) # pick some really late time
rcd = restore.RestoreCombinedData(rid, rpbase, rptarget)
rcd.RestoreFile()
#sorted_incs = Restore.sortincseq(rest_time, incs)
#Restore.RestoreFile(rest_time, rpbase, (), sorted_incs, rptarget)
rptarget.setdata()
if not rptarget.lstat(): assert not pair[1].lstat()
elif not pair[1].lstat(): assert not rptarget.lstat()
else:
assert rpath.cmp(rptarget, pair[1]), \
"%s %s" % (rptarget.path, pair[1].path)
assert rpath.cmp_attribs(rptarget, pair[1]), \
"%s %s" % (rptarget.path, pair[1].path)
rptarget.delete()
def testsortincseq(self):
"""Test the Restore.sortincseq function
This test just makes sure that it comes up with the right
number of increments for each base name - given a list of
increments, we should eventually get sorted sequences that
end in each one (each one will be the last increment once).
"""
for basename in ['ocaml', 'mf']:
tuples, unused = self.maketesttuples(basename)
incs = [tuple[0] for tuple in tuples]
# Now we need a time newer than any inc
mirror_time = Time.stringtotime(incs[-1].getinctime()) + 10000
for inc, incbase in tuples:
assert inc.isincfile()
inctime = Time.stringtotime(inc.getinctime())
rid1 = restore.RestoreIncrementData(basename, incbase, incs)
rid1.sortincseq(inctime, mirror_time)
assert rid1.inc_list, rid1.inc_list
# oldest increment should be exactly inctime
ridtime = Time.stringtotime(rid1.inc_list[-1].getinctime())
assert ridtime == inctime, (ridtime, inctime)
def testRestorefiles(self):
"""Testing restoration of files one at a time"""
map(self.restoreonefiletest, ["ocaml", "mf"])
def testRestoreDir(self):
"""Test restoring from a real backup set
Run makerestoretest3 if this doesn't work.
"""
Myrm("testfiles/output")
InternalRestore(1, 1, "testfiles/restoretest3",
"testfiles/output", 20000)
src_rp = rpath.RPath(Globals.local_connection, "testfiles/increment2")
restore_rp = rpath.RPath(Globals.local_connection, "testfiles/output")
assert CompareRecursive(src_rp, restore_rp)
def testRestoreCorrupt(self):
"""Test restoring a partially corrupt archive
The problem here is that a directory is missing from what is
to be restored, but because the previous backup was aborted in
the middle, some of the files in that directory weren't marked
as .missing.
def get_rfcs(self):
"""Return available RestoreFileCompararer objects"""
base_rf = restore.RestoreFile(restore_base_rp, restore_base_rp, [])
rfs = base_rf.yield_sub_rfs()
rfcs = []
for rf in rfs:
if rf.mirror_rp.dirsplit()[1] in ["dir"]:
log.Log("skipping 'dir'", 5)
continue
rfc = RestoreFileComparer(rf)
for inc in rf.inc_list:
test_time = inc.getinctime()
rfc.add_rpath(self.get_correct(rf.mirror_rp, test_time),
test_time)
rfc.add_rpath(rf.mirror_rp, mirror_time)
rfcs.append(rfc)
return rfcs
def get_correct(self, mirror_rp, test_time):
"""Return correct version with base mirror_rp at time test_time"""
assert -1 < test_time < 2000000000, test_time
dirname, basename = mirror_rp.dirsplit()
for filename in restore_base_filenames:
comps = filename.split(".")
base = ".".join(comps[:-1])
t = Time.stringtotime(comps[-1])
if t == test_time and basename == base:
return restore_base_rp.append(filename)
# Correct rp must be empty
return restore_base_rp.append("%s.%s" %
(basename, Time.timetostring(test_time)))
def testRestoreSingle(self):
"""Test restoring files one at at a time"""
MakeOutputDir()
for rfc in self.get_rfcs():
if rfc.rf.inc_rp.isincfile(): continue
log.Log("Comparing %s" % (rfc.rf.inc_rp.path,), 5)
rfc.compare_all()
def testBothLocal(self):
"""Test directory restore everything local"""
self.restore_dir_test(1,1)
def testMirrorRemote(self):
"""Test directory restore mirror is remote"""
self.restore_dir_test(0, 1)
def testDestRemote(self):
"""Test directory restore destination is remote"""
self.restore_dir_test(1, 0)
def testBothRemote(self):
"""Test directory restore everything is remote"""
self.restore_dir_test(0, 0)
def restore_dir_test(self, mirror_local, dest_local):
"""Run whole dir tests
If any of the above tests don't work, try rerunning
makerestoretest3.
"""
Myrm("testfiles/output")
InternalRestore(1, 1, "testfiles/restoretest4", "testfiles/output",
10000)
assert os.lstat("testfiles/output")
self.assertRaises(OSError, os.lstat, "testfiles/output/tmp")
self.assertRaises(OSError, os.lstat, "testfiles/output/rdiff-backup")
target_rp = rpath.RPath(Globals.local_connection, "testfiles/output")
mirror_rp = rpath.RPath(Globals.local_connection,
"testfiles/restoretest3")
inc1_rp = rpath.RPath(Globals.local_connection,
"testfiles/increment1")
inc2_rp = rpath.RPath(Globals.local_connection,
"testfiles/increment2")
inc3_rp = rpath.RPath(Globals.local_connection,
"testfiles/increment3")
inc4_rp = rpath.RPath(Globals.local_connection,
"testfiles/increment4")
InternalRestore(mirror_local, dest_local, "testfiles/restoretest3",
"testfiles/output", 45000)
assert CompareRecursive(inc4_rp, target_rp)
InternalRestore(mirror_local, dest_local, "testfiles/restoretest3",
"testfiles/output", 35000)
assert CompareRecursive(inc3_rp, target_rp, compare_hardlinks = 0)
InternalRestore(mirror_local, dest_local, "testfiles/restoretest3",
"testfiles/output", 25000)
assert CompareRecursive(inc2_rp, target_rp, compare_hardlinks = 0)
InternalRestore(mirror_local, dest_local, "testfiles/restoretest3",
"testfiles/output", 5000)
assert CompareRecursive(inc1_rp, target_rp, compare_hardlinks = 0)
# def testRestoreCorrupt(self):
# """Test restoring a partially corrupt archive
#
# The problem here is that a directory is missing from what is
# to be restored, but because the previous backup was aborted in
# the middle, some of the files in that directory weren't marked
# as .missing.
#
# """
# Myrm("testfiles/output")
# InternalRestore(1, 1, "testfiles/restoretest4", "testfiles/output",
# 10000)
# assert os.lstat("testfiles/output")
# self.assertRaises(OSError, os.lstat, "testfiles/output/tmp")
# self.assertRaises(OSError, os.lstat, "testfiles/output/rdiff-backup")
def testRestoreNoincs(self):
"""Test restoring a directory with no increments, just mirror"""
......
trunk/rdiff-backup/testing/robusttest.py
View file @ fe98b295
import os, unittest
from commontest import *
from rdiff_backup import rpath, robust, TempFile, Globals
class TestRobustAction(unittest.TestCase):
"""Test some robust actions"""
def testCopyWithAttribs(self):
"""Test copy with attribs action"""
rpin = rpath.RPath(Globals.local_connection, "./testfiles/robust/in")
fp = open("./testfiles/robust/in", "wb")
fp.write("hello there")
fp.close()
os.chmod("./testfiles/robust/in", 0604)
rpin.setdata()
assert rpin.isreg() and rpin.getperms() % 01000 == 0604
rpout = rpath.RPath(Globals.local_connection, "./testfiles/robust/out")
robust.copy_with_attribs_action(rpin, rpout).execute()
if not rpout == rpin:
print rpout, rpin
assert 0
rpout.delete()
rpin.delete()
class TempFileTest(unittest.TestCase):
"""Test creation and management of tempfiles"""
"""Test creation and management of tempfiles in TempFile module"""
rp_base = rpath.RPath(Globals.local_connection,
"./testfiles/robust/testfile_base")
def testBasic(self):
......@@ -61,26 +40,19 @@ class TempFileTest(unittest.TestCase):
assert destination.lstat()
destination.delete()
class SaveStateTest(unittest.TestCase):
"""Test SaveState class"""
data_dir = rpath.RPath(Globals.local_connection, "testfiles/robust")
def testSymlinking(self):
"""Test recording last file with symlink"""
last_rorp = rpath.RORPath(('usr', 'local', 'bin', 'ls'))
Globals.rbdir = self.data_dir
Time.setcurtime()
SetConnections.BackupInitConnections(Globals.local_connection,
Globals.local_connection)
robust.SaveState.init_filenames()
robust.SaveState.record_last_file_action(last_rorp).execute()
sym_rp = rpath.RPath(Globals.local_connection,
"testfiles/robust/last-file-incremented.%s.data" %
Time.curtimestr)
assert sym_rp.issym()
assert sym_rp.readlink() == "increments/usr/local/bin/ls"
sym_rp.delete()
class RobustTest(unittest.TestCase):
"""Test robust module"""
def test_check_common_error(self):
"""Test capturing errors"""
def cause_catchable_error(a):
os.lstat("aoenuthaoeu/aosutnhcg.4fpr,38p")
def cause_uncatchable_error():
ansoethusaotneuhsaotneuhsaontehuaou
result = robust.check_common_error(None, cause_catchable_error, [1])
assert result is None, result
try: robust.check_common_error(None, cause_uncatchable_error)
except NameError: pass
else: assert 0, "Key error not raised"
if __name__ == '__main__': unittest.main()
trunk/rdiff-backup/testing/roottest.py
View file @ fe98b295
import unittest, os
from commontest import *
from rdiff_backup.log import *
from rdiff_backup import Globals
from rdiff_backup import Globals, log
"""Root tests
......@@ -11,7 +10,11 @@ that are meant to be run as root.
Globals.set('change_source_perms', None)
Globals.counter = 0
Log.setverbosity(4)
log.Log.setverbosity(4)
def Run(cmd):
print "Running: ", cmd
assert not os.system(cmd)
class RootTest(unittest.TestCase):
dirlist1 = ["testfiles/root", "testfiles/various_file_types", "testfiles/increment4"]
......@@ -21,8 +24,63 @@ class RootTest(unittest.TestCase):
def testLocal2(self): BackupRestoreSeries(1, 1, self.dirlist2)
def testRemote(self): BackupRestoreSeries(None, None, self.dirlist1)
def tearDown(self):
os.system(MiscDir + "/myrm testfiles/output testfiles/rest_out")
class NonRoot(unittest.TestCase):
"""Test backing up as non-root user
Test backing up a directory with files of different userids and
with device files in it, as a non-root user. When restoring as
root, everything should be restored normally.
"""
user = 'ben'
def make_root_dir(self):
"""Make directory createable only by root"""
rp = rpath.RPath(Globals.local_connection, "testfiles/root_out")
if rp.lstat(): Myrm(rp.path)
rp.mkdir()
rp1 = rp.append("1")
rp1.touch()
rp2 = rp.append("2")
rp2.touch()
rp2.chown(1, 1)
rp3 = rp.append("3")
rp3.touch()
rp3.chown(2, 2)
rp4 = rp.append("dev")
rp4.makedev('c', 4, 28)
return rp
def test_non_root(self):
"""Main non-root -> root test"""
Myrm("testfiles/output")
input_rp = self.make_root_dir()
Globals.change_ownership = 1
output_rp = rpath.RPath(Globals.local_connection, "testfiles/output")
restore_rp = rpath.RPath(Globals.local_connection,
"testfiles/rest_out")
empty_rp = rpath.RPath(Globals.local_connection, "testfiles/empty")
backup_cmd = "rdiff-backup %s %s" % (input_rp.path, output_rp.path)
Run("su %s -c '%s'" % (self.user, backup_cmd))
Myrm("testfiles/rest_out")
restore_cmd = "rdiff-backup -r now %s %s" % (output_rp.path,
restore_rp.path,)
Run(restore_cmd)
assert CompareRecursive(input_rp, restore_rp)
backup_cmd = "rdiff-backup %s %s" % (empty_rp.path, output_rp.path)
Run("su %s -c '%s'" % (self.user, backup_cmd))
Myrm("testfiles/rest_out")
Run(restore_cmd)
assert CompareRecursive(empty_rp, restore_rp)
Myrm("testfiles/rest_out")
restore_cmd = "rdiff-backup -r 1 %s %s" % (output_rp.path,
restore_rp.path,)
Run(restore_cmd)
assert CompareRecursive(input_rp, restore_rp)
if __name__ == "__main__": unittest.main()
trunk/rdiff-backup/testing/rorpitertest.py
View file @ fe98b295
......@@ -52,25 +52,6 @@ class RORPIterTest(unittest.TestCase):
iter([])))
def testCombinedPatching(self):
"""Combined signature, patch, and diff operations"""
if self.output.lstat():
Myrm(self.output.path)
self.output.setdata()
def turninto(final_rp):
sigfile = rorpiter.ToFile(rorpiter.GetSignatureIter(self.output))
diff_file = rorpiter.ToFile(rorpiter.GetDiffIter(
rorpiter.FromFile(sigfile), rorpiter.IterateRPaths(final_rp)))
rorpiter.PatchIter(self.output, rorpiter.FromFile(diff_file))
turninto(self.inc1rp)
rpath.copy_attribs(self.inc1rp, self.output) # Update time
assert self.compare_no_times(self.inc1rp, self.output)
turninto(self.inc2rp)
rpath.copy_attribs(self.inc2rp, self.output)
assert self.compare_no_times(self.inc2rp, self.output)
def compare_no_times(self, src_rp, dest_rp):
"""Compare but disregard directories attributes"""
def equal(src_rorp, dest_rorp):
......
trunk/rdiff-backup/testing/selectiontest.py
View file @ fe98b295
......@@ -417,19 +417,19 @@ testfiles/select**/2
("--exclude", "/")],
[(), ("home",)])
def testParseStartingFrom(self):
"""Test parse, this time starting from inside"""
self.root = rpath.RPath(Globals.local_connection, "testfiles/select")
self.Select = Select(self.root)
self.Select.ParseArgs([("--include", "testfiles/select/1/1"),
("--exclude", "**")], [])
self.Select.set_iter(('1', '1'))
assert lazy.Iter.equal(lazy.Iter.map(lambda dsrp: dsrp.index,
self.Select),
iter([("1", '1', '1'),
('1', '1', '2'),
('1', '1', '3')]),
verbose = 1)
# def testParseStartingFrom(self):
# """Test parse, this time starting from inside"""
# self.root = rpath.RPath(Globals.local_connection, "testfiles/select")
# self.Select = Select(self.root)
# self.Select.ParseArgs([("--include", "testfiles/select/1/1"),
# ("--exclude", "**")], [])
# self.Select.set_iter(('1', '1'))
# assert lazy.Iter.equal(lazy.Iter.map(lambda dsrp: dsrp.index,
# self.Select),
# iter([("1", '1', '1'),
# ('1', '1', '2'),
# ('1', '1', '3')]),
# verbose = 1)
if __name__ == "__main__": unittest.main()
trunk/rdiff-backup/testing/statisticstest.py
View file @ fe98b295
import unittest
import unittest, time
from commontest import *
from rdiff_backup import statistics, rpath
from rdiff_backup import statistics, rpath, restore
class StatsObjTest(unittest.TestCase):
"""Test StatsObj class"""
......@@ -29,7 +29,7 @@ class StatsObjTest(unittest.TestCase):
self.set_obj(s)
assert s.get_stat('SourceFiles') == 1
s1 = statistics.ITRB()
s1 = statistics.StatFileObj()
assert s1.get_stat('SourceFiles') == 0
def test_get_stats_string(self):
......@@ -40,10 +40,12 @@ class StatsObjTest(unittest.TestCase):
self.set_obj(s)
stats_string = s.get_stats_string()
assert stats_string == \
"""StartTime 11.00 (Wed Dec 31 16:00:11 1969)
EndTime 12.00 (Wed Dec 31 16:00:12 1969)
ElapsedTime 1.00 (1 second)
ss_list = stats_string.split("\n")
tail = "\n".join(ss_list[2:]) # Time varies by time zone, don't check
#"""StartTime 11.00 (Wed Dec 31 16:00:11 1969)
#EndTime 12.00 (Wed Dec 31 16:00:12 1969)"
assert tail == \
"""ElapsedTime 1.00 (1 second)
SourceFiles 1
SourceFileSize 2 (2 bytes)
MirrorFiles 13
......@@ -143,4 +145,81 @@ TotalDestinationSizeChange 7 (7 bytes)
assert s3.SourceFiles == 75
class IncStatTest(unittest.TestCase):
"""Test statistics as produced by actual backup"""
def stats_check_initial(self, s):
"""Make sure stats object s compatible with initial mirroring
A lot of the off by one stuff is because the root directory
exists in the below examples.
"""
assert s.MirrorFiles == 1 or s.MirrorFiles == 0
assert s.MirrorFileSize < 20000
assert s.NewFiles <= s.SourceFiles <= s.NewFiles + 1
assert s.NewFileSize <= s.SourceFileSize <= s.NewFileSize + 20000
assert s.ChangedFiles == 1 or s.ChangedFiles == 0
assert s.ChangedSourceSize < 20000
assert s.ChangedMirrorSize < 20000
assert s.DeletedFiles == s.DeletedFileSize == 0
assert s.IncrementFileSize == 0
def testStatistics(self):
"""Test the writing of statistics
The file sizes are approximate because the size of directories
could change with different file systems...
"""
Globals.compression = 1
Myrm("testfiles/output")
InternalBackup(1, 1, "testfiles/stattest1", "testfiles/output")
InternalBackup(1, 1, "testfiles/stattest2", "testfiles/output",
time.time()+1)
rbdir = rpath.RPath(Globals.local_connection,
"testfiles/output/rdiff-backup-data")
#incs = Restore.get_inclist(rbdir.append("subdir").
# append("directory_statistics"))
#assert len(incs) == 2
#s1 = StatsObj().read_stats_from_rp(incs[0]) # initial mirror stats
#assert s1.SourceFiles == 2
#assert 400000 < s1.SourceFileSize < 420000
#self.stats_check_initial(s1)
#subdir_stats = StatsObj().read_stats_from_rp(incs[1]) # increment stats
#assert subdir_stats.SourceFiles == 2
#assert 400000 < subdir_stats.SourceFileSize < 420000
#assert subdir_stats.MirrorFiles == 2
#assert 400000 < subdir_stats.MirrorFileSize < 420000
#assert subdir_stats.NewFiles == subdir_stats.NewFileSize == 0
#assert subdir_stats.DeletedFiles == subdir_stats.DeletedFileSize == 0
#assert subdir_stats.ChangedFiles == 2
#assert 400000 < subdir_stats.ChangedSourceSize < 420000
#assert 400000 < subdir_stats.ChangedMirrorSize < 420000
#assert 10 < subdir_stats.IncrementFileSize < 20000
incs = restore.get_inclist(rbdir.append("session_statistics"))
assert len(incs) == 2
s2 = statistics.StatsObj().read_stats_from_rp(incs[0])
assert s2.SourceFiles == 7
assert 700000 <= s2.SourceFileSize < 750000
self.stats_check_initial(s2)
root_stats = statistics.StatsObj().read_stats_from_rp(incs[1])
assert root_stats.SourceFiles == 7, root_stats.SourceFiles
assert 550000 <= root_stats.SourceFileSize < 570000
assert root_stats.MirrorFiles == 7
assert 700000 <= root_stats.MirrorFileSize < 750000
assert root_stats.NewFiles == 1
assert root_stats.NewFileSize == 0
assert root_stats.DeletedFiles == 1
assert root_stats.DeletedFileSize == 200000
assert 3 <= root_stats.ChangedFiles <= 4, root_stats.ChangedFiles
assert 450000 <= root_stats.ChangedSourceSize < 470000
assert 400000 <= root_stats.ChangedMirrorSize < 420000, \
root_stats.ChangedMirrorSize
assert 10 < root_stats.IncrementFileSize < 30000
if __name__ == "__main__": unittest.main()
trunk/rdiff-backup/testing/timetest.py
View file @ fe98b295
......@@ -6,7 +6,7 @@ class TimeTest(unittest.TestCase):
def testConversion(self):
"""test timetostring and stringtotime"""
Time.setcurtime()
assert type(Time.curtime) is types.FloatType
assert type(Time.curtime) is types.FloatType or types.LongType
assert type(Time.curtimestr) is types.StringType
assert (Time.cmp(int(Time.curtime), Time.curtimestr) == 0 or
Time.cmp(int(Time.curtime) + 1, Time.curtimestr) == 0)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7