|
From: <di...@us...> - 2011-10-13 19:40:39
|
Revision: 762
http://safekeep.svn.sourceforge.net/safekeep/?rev=762&view=rev
Author: dimi
Date: 2011-10-13 19:40:33 +0000 (Thu, 13 Oct 2011)
Log Message:
-----------
Replace object identies test ('is') with value comparisons ('==').
Based on a suggestion by Harald Nehring <har...@ar...>
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2011-09-30 00:22:02 UTC (rev 761)
+++ safekeep/trunk/safekeep 2011-10-13 19:40:33 UTC (rev 762)
@@ -278,7 +278,7 @@
fin.close()
for line in lines:
line = line.strip()
- if len(line) is 0 or line[0] is '#': continue
+ if len(line) == 0 or line[0] == '#': continue
if '=' in line:
key, value = line.split('=', 1)
props[key.strip()] = value.strip()
@@ -617,7 +617,7 @@
lines.reverse()
for line in lines:
matches = pattern.match(line)
- if not matches is None:
+ if matches is not None:
mounts.append(matches.groups())
return mounts
@@ -921,7 +921,7 @@
ionice_out = try_to_run([ionice_cmd, '-h'])
if ionice_out is not None:
ionice_args = []
- if ionice is 'idle':
+ if ionice == 'idle':
ionice_args.extend(['-c3'])
else:
ionice_args.extend(['-c2', '-n%s' % (ionice)])
@@ -1157,13 +1157,13 @@
args = ['rdiff-backup']
- if list_type is 'increments':
+ if list_type == 'increments':
args.extend(['--list-increments'])
- elif list_type is 'sizes':
+ elif list_type == 'sizes':
args.extend(['--list-increment-sizes'])
- elif list_type is 'changed':
+ elif list_type == 'changed':
args.extend(['--list-changed-since', list_date])
- elif list_type is 'attime':
+ elif list_type == 'attime':
args.extend(['--list-at-time', list_date])
else:
assert False, 'Unknown list type: ' + list_type
@@ -1214,7 +1214,7 @@
if deploy:
info('%s: Key do not exist, generating it now: %s' % (id, privatekeyfile))
gencmd = 'ssh-keygen -q -b 1024 -t dsa -N "" -C "SafeKeep auto generated key at %s@%s" -f %s' % (backup_user, os.uname()[1], privatekeyfile)
- if backup_user is not work_user:
+ if backup_user != work_user:
gencmd = 'su -s /bin/sh -c %s - %s' % (commands.mkarg(gencmd), backup_user)
debug(gencmd)
if spawn(gencmd):
@@ -1285,13 +1285,13 @@
for i, c in enumerate(line):
if in_str:
if in_esc: in_esc = False
- elif c is '\'': in_esc = True
- elif c is '"': in_str = False
+ elif c == '\'': in_esc = True
+ elif c == '"': in_str = False
else:
- if c is ' ':
+ if c == ' ':
rest = line[i:].strip()
break
- elif c is '"': in_str = True
+ elif c == '"': in_str = True
opts += c
else:
info('Invalid key line, ignoring: %s' % line)
@@ -1315,7 +1315,7 @@
base46enc = parts[1]
- if len(parts) is 2:
+ if len(parts) == 2:
comment = None
else:
comment = parts[2]
@@ -1470,22 +1470,22 @@
if mode is None:
usage(2)
- if mode is not 'keys' and (identity or keys_status or keys_print or keys_deploy):
+ if mode != 'keys' and (identity or keys_status or keys_print or keys_deploy):
usage(2)
- if mode is not 'list' and (list_type or list_date or list_parsable):
+ if mode != 'list' and (list_type or list_date or list_parsable):
usage(2)
- if mode is not 'server' and (email or smtp):
+ if mode != 'server' and (email or smtp):
usage(2)
if not mode in ['server', 'client'] and cleanup:
usage(2)
- if mode is 'client' and cfglocs:
+ if mode == 'client' and cfglocs:
usage(2)
- if mode is not 'client':
+ if mode != 'client':
if cfgfile is None and os.path.isfile(config_file):
cfgfile = config_file
if cfgfile and os.path.isfile(cfgfile):
@@ -1499,7 +1499,7 @@
def get_int(p):
v = props.get(p)
- if v is not None and v is not '':
+ if v is not None and v != '':
return int(v)
return None
@@ -1519,7 +1519,7 @@
nice_cln = get_int('nice.adjustment.client') or nice_def
ionice_def = props.get('ionice.adjustment')
if ionice_def is None: ionice_def = 'idle'
- if ionice_def is '': ionice_def = 'none'
+ if ionice_def == '': ionice_def = 'none'
global default_bandwidth
default_bandwidth['overall'] = get_int('bandwidth.limit') or 0
@@ -1532,7 +1532,7 @@
if backup_user and backup_user != work_user:
(user, pswd, uid, gid, gecos, home_dir, shell) = pwd.getpwnam(backup_user)
- if mode is not 'keys':
+ if mode != 'keys':
try:
os.setregid(gid, gid)
os.setreuid(uid, uid)
@@ -1551,7 +1551,7 @@
else:
cfgs = {}
- if mode is 'client':
+ if mode == 'client':
if len(args) > 0: usage(2)
else:
ok = True
@@ -1568,17 +1568,17 @@
if verbosity > 2:
verbosity_trickle = verbosity_ssh = '-' + (verbosity-2) * 'v'
- if mode is 'server':
+ if mode == 'server':
is_client = False
verbosity_level = 1 + verbosity
do_server(cfgs, args, nice_srv, ionice_def, force, cleanup)
- elif mode is 'list':
+ elif mode == 'list':
if list_type is None:
list_type = 'increments'
is_client = False
verbosity_level = 2 + verbosity
do_list(cfgs, args, list_type, list_date, list_parsable)
- elif mode is 'client':
+ elif mode == 'client':
if cleanup:
is_client = False
verbosity_level = 1 + verbosity
@@ -1587,7 +1587,7 @@
is_client = True
verbosity_level = 3 + verbosity
do_client()
- elif mode is 'keys':
+ elif mode == 'keys':
is_client = False
verbosity_level = 1 + verbosity
if not keys_status and not keys_print and not keys_deploy:
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <di...@us...> - 2011-10-16 15:39:26
|
Revision: 763
http://safekeep.svn.sourceforge.net/safekeep/?rev=763&view=rev
Author: dimi
Date: 2011-10-16 15:39:19 +0000 (Sun, 16 Oct 2011)
Log Message:
-----------
Frank Crawford <fr...@cr...>
It "fixes" the issues with the removal of snapshots, by
executing "dmsetup remove" prior to the actual lvremove. This appears
to be current best practice, although I've noticed there are some
proposed patches for an upcoming lvm2 update that may fix the issue
fully.
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2011-10-13 19:40:33 UTC (rev 762)
+++ safekeep/trunk/safekeep 2011-10-16 15:39:19 UTC (rev 763)
@@ -639,6 +639,17 @@
return (group, volume, mountpoint, mounttype)
return (None, None, None, None)
+def do_lvremove(device):
+ (group, volume) = device.split('/')[-2:]
+ if group == 'mapper':
+ lvmdev = device;
+ else:
+ lvmdev = '/dev/mapper/%s-%s' % (group, volume.replace('-', '--'))
+ ret = spawn(['dmsetup', 'remove', lvmdev])
+ ret = spawn(['dmsetup', 'remove', lvmdev + '-cow'])
+ ret = spawn(['lvremove', '--force', device])
+ return ret
+
def gather_snap_information(device, bdir):
(group, volume, mountpoint, mounttype) = gather_lvm_information(device)
if not mountpoint: return (None, None, None, None)
@@ -669,7 +680,7 @@
ec = spawn(args)
if ec:
warn('Can not mount the snapshot: %s' % device)
- ret = spawn(['lvremove', '--force', snapdev])
+ ret = do_lvremove(snapdev)
if ret:
warn('Can not tear down snapshot: %s' % device)
@@ -688,12 +699,7 @@
if ret:
warn('Can not umount the snapshot: %s' % snapmnt)
- # stupid workaround for https://bugzilla.redhat.com/show_bug.cgi?id=577798
- for i in range(1,10):
- ret = spawn(['lvremove', '--force', snapdev])
- if not ret:
- break
-
+ ret = do_lvremove(snapdev)
if ret:
warn('Can not tear down snapshot: %s' % device)
@@ -809,7 +815,7 @@
warn('Can not unmount the snapshot: %s' % mountpoint)
if fnmatch.fnmatch(device, '*_snap_safekeep-*'):
info("Removing snapshot %s" % device)
- ret = spawn(['lvremove', '--force', device])
+ ret = do_lvremove(device)
if ret:
warn('Can not tear down snapshot: %s' % device)
scrubbed = True
@@ -820,7 +826,7 @@
for (volume, group) in lvm_snap_information():
device = os.path.join('/dev', group, volume)
info("Removing snapshot %s" % device)
- ret = spawn(['lvremove', '--force', device])
+ ret = do_lvremove(device)
if ret:
warn('Can not tear down snapshot: %s' % device)
scrubbed = True
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <di...@us...> - 2011-10-16 15:40:47
|
Revision: 764
http://safekeep.svn.sourceforge.net/safekeep/?rev=764&view=rev
Author: dimi
Date: 2011-10-16 15:40:41 +0000 (Sun, 16 Oct 2011)
Log Message:
-----------
Frank Crawford <fr...@cr...>
- Changes the --list option to continue even if one of the hosts gives
an error,
- Stop printing the separator if nothing was printed earlier, and
- Clean up some definitions to use True and False rather than 0 and 1
for variable that are booleans.
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2011-10-16 15:39:19 UTC (rev 763)
+++ safekeep/trunk/safekeep 2011-10-16 15:40:41 UTC (rev 764)
@@ -1039,13 +1039,15 @@
def do_server(cfgs, ids, nice, ionice, force, cleanup):
debug("Do server main loop")
+ output_done = False
for cfg in cfgs.itervalues():
id = cfg['id']
if ids and id not in ids: continue
info('------------------------------------------------------------------')
info('Server backup starting for client %s' % id)
+ output_done = True
- cleaned_up = 0
+ cleaned_up = False
try:
if cfg['host']:
if not os.path.isfile(cfg['key_ctrl']):
@@ -1093,7 +1095,7 @@
do_server_getanswer(cout)
bdir = '/' # Fake directory for the rest of the cleanup
do_server_rdiff_cleanup(cfg)
- cleaned_up = 1
+ cleaned_up = True
errs = 0
else:
cin.write('SETUP\n')
@@ -1146,11 +1148,13 @@
else:
error('Server backup for client %s: FAILED' % id, ex)
- info('------------------------------------------------------------------')
+ if output_done:
+ info('------------------------------------------------------------------')
debug('Server backup done')
def do_list(cfgs, ids, list_type, list_date, list_parsable):
debug("Do server listing main loop")
+ output_done = False
for cfg in cfgs.itervalues():
id = cfg['id']
if ids and id not in ids: continue
@@ -1159,7 +1163,7 @@
else:
info('------------------------------------------------------------------')
info('Server listing for client %s' % id)
-
+ output_done = True
args = ['rdiff-backup']
@@ -1180,9 +1184,9 @@
args.extend([cfg['dir']])
ret = spawn(args)
if ret:
- raise Exception('Failed to run rdiff-backup')
+ error('Failed to run rdiff-backup')
- if not list_parsable:
+ if output_done and not list_parsable:
info('------------------------------------------------------------------')
debug('Server listing done')
@@ -1392,16 +1396,16 @@
cfglocs = []
verbosity = 0
clientid = None
- force = 0
- cleanup = 0
- noemail = 0
+ force = False
+ cleanup = False
+ noemail = False
list_type = None
- list_parsable = 0
+ list_parsable = False
list_date = None
identity = None
- keys_status = None
- keys_print = None
- keys_deploy = None
+ keys_status = False
+ keys_print = False
+ keys_deploy = False
nice_srv = None
for o, a in opts:
if o in ('-c', '--conf'):
@@ -1436,11 +1440,11 @@
if mode: usage(2)
mode = 'keys'
elif o in ('--force', ):
- force = 1
+ force = True
elif o in ('--cleanup', ):
- cleanup = 1
+ cleanup = True
elif o in ('--noemail', ):
- noemail = 1
+ noemail = True
elif o in ('--increments', ):
if list_type: usage(2)
list_type = 'increments'
@@ -1448,7 +1452,7 @@
if list_type: usage(2)
list_type = 'sizes'
elif o in ('--parsable-output', ):
- list_parsable = 1
+ list_parsable = True
elif o in ('--changed', ):
if list_type: usage(2)
list_type = 'changed'
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <di...@us...> - 2011-10-20 14:42:21
|
Revision: 765
http://safekeep.svn.sourceforge.net/safekeep/?rev=765&view=rev
Author: dimi
Date: 2011-10-20 14:42:14 +0000 (Thu, 20 Oct 2011)
Log Message:
-----------
Frank Crawford <fr...@cr...>
More reliable snapshot removal.
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2011-10-16 15:40:41 UTC (rev 764)
+++ safekeep/trunk/safekeep 2011-10-20 14:42:14 UTC (rev 765)
@@ -645,9 +645,13 @@
lvmdev = device;
else:
lvmdev = '/dev/mapper/%s-%s' % (group, volume.replace('-', '--'))
- ret = spawn(['dmsetup', 'remove', lvmdev])
- ret = spawn(['dmsetup', 'remove', lvmdev + '-cow'])
- ret = spawn(['lvremove', '--force', device])
+ for i in range(1,10):
+ ret = spawn(['sync'])
+ ret = spawn(['dmsetup', 'remove', lvmdev])
+ ret = spawn(['dmsetup', 'remove', lvmdev + '-cow'])
+ ret = spawn(['lvremove', '--force', device])
+ if not ret:
+ break
return ret
def gather_snap_information(device, bdir):
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <fcr...@us...> - 2011-11-20 05:35:37
|
Revision: 766
http://safekeep.svn.sourceforge.net/safekeep/?rev=766&view=rev
Author: fcrawford
Date: 2011-11-20 05:35:31 +0000 (Sun, 20 Nov 2011)
Log Message:
-----------
Update version
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2011-10-20 14:42:14 UTC (rev 765)
+++ safekeep/trunk/safekeep 2011-11-20 05:35:31 UTC (rev 766)
@@ -68,7 +68,7 @@
cmd = "<Missing>"
PROTOCOL = "1.1"
-VERSION = "1.3.2"
+VERSION = "1.3.3"
VEBOSITY_BY_CLASS = {'DBG': 3, 'INFO': 2, 'WARN': 1, 'ERR': 0}
######################################################################
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <fcr...@us...> - 2011-11-28 12:38:43
|
Revision: 775
http://safekeep.svn.sourceforge.net/safekeep/?rev=775&view=rev
Author: fcrawford
Date: 2011-11-28 12:38:37 +0000 (Mon, 28 Nov 2011)
Log Message:
-----------
- includes the PID of the creating safekeep in the snapshot names,
- handles the client side cleanup, if snapshots are left around,
- cleans up the backup files if the client fails,
- intentionally block run a "safekeep --client --cleanup" when other safekeep is running.
- clean up of messages and some control flow,
- suppress tracebacks, only printing them on debug requests,
- ensure that the client is closed before starting some possibly long running server processes.
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2011-11-28 12:38:02 UTC (rev 774)
+++ safekeep/trunk/safekeep 2011-11-28 12:38:37 UTC (rev 775)
@@ -64,6 +64,7 @@
backup_user = None
home_dir = None
base_dir = None
+current_pid = os.getpid()
default_bandwidth = {}
cmd = "<Missing>"
@@ -140,7 +141,7 @@
def error(msg, ex=None):
extra = ""
- if ex:
+ if ex and verbosity_level > 2:
extra = stacktrace()
log(msg + extra, 'ERR')
@@ -595,7 +596,7 @@
continue
try:
os.remove(dump['file'])
- except Exception, e:
+ except OSError, e:
warn('Unable to remove dump file: %s for database %s because: %s' %
(dump['file'], dump['db'], e))
@@ -738,20 +739,20 @@
debug('Checking FS snapshots')
for snap in cfg['snaps']:
device = snap['device']
- if check_lvm_information(device):
- raise Exception("Previous snapshots found for %s: run 'safekeep --server --cleanup' to correct" % device)
+ if check_lvm_information(device) and not do_client_scrub():
+ raise Exception("Previous snapshots found for %s and automatic correction failed: run 'safekeep --server --cleanup' to correct" % device)
ret = spawn(['modprobe', 'dm-snapshot'])
if ret:
warn('modprobe dm-snapshot failed, continuing')
- bdir = tempfile.mkdtemp("-rbind", "safekeep-", "/mnt")
+ bdir = tempfile.mkdtemp("-rbind", "safekeep-%d-" % current_pid, "/mnt")
ret = spawn(['mount', '--rbind', '/', bdir])
if ret:
warn('mount --rbind failed, snapshotting will be disabled')
try:
os.rmdir(bdir)
- except Exception, e:
- warn('Failed to remove: %s' % bdir)
+ except OSError, e:
+ warn('Failed to remove: %s: %s' % (bdir, e))
bdir = '/'
else:
do_client_snap(cfg, bdir)
@@ -772,8 +773,8 @@
else:
try:
os.rmdir(bdir)
- except Exception, e:
- warn('Unable to remove: ' + bdir)
+ except OSError, e:
+ warn('Unable to remove: %s: %s' % (bdir, e))
do_client_dbdump_teardown(cfg)
@@ -792,6 +793,22 @@
else:
scrubbed = False
+ # Go through and see if any come from existing safekeep processes
+ pattern = re.compile(r"_snap_safekeep-(\d+)-")
+ lvm_snap_list = lvm_snap_information()
+ for (volume, group) in lvm_snap_list:
+ matches = pattern.search(volume)
+ if matches is not None:
+ pid = matches.group(1)
+ # Look up /proc/<pid>/cmdline to see what process is running
+ proc_file = "/proc/" + pid + "/cmdline"
+ if pid != current_pid and os.path.exists(proc_file):
+ fin = open(proc_file, "r")
+ (cmd, arg0, args) = fin.read().split('\0', 2)
+ fin.close()
+ if os.path.basename(arg0) == "safekeep":
+ raise Exception('another safekeep process running: pid %s' % pid)
+
if os.environ['PATH'][-1] == ':':
os.environ['PATH'] += '/sbin:/usr/sbin:/usr/local/sbin:'
else:
@@ -811,8 +828,8 @@
else:
try:
os.rmdir(mountpoint)
- except Exception, e:
- warn('Failed to remove: %s' % mountpoint)
+ except OSError, e:
+ warn('Failed to remove: %s: %s' % (mountpoint, e))
else:
ret = spawn(['umount', mountpoint])
if ret:
@@ -827,7 +844,7 @@
# Now cleanup any snapshots still hanging around
debug("Cleaning up remaining snapshots")
- for (volume, group) in lvm_snap_information():
+ for (volume, group) in lvm_snap_list:
device = os.path.join('/dev', group, volume)
info("Removing snapshot %s" % device)
ret = do_lvremove(device)
@@ -845,12 +862,19 @@
info("Removing rbind directory %s" % mountpoint)
try:
os.rmdir(mountpoint)
- except Exception, e:
- warn('Failed to remove: %s' % mountpoint)
+ except OSError, e:
+ warn('Failed to remove: %s: %s' % (mountpoint, e))
if not scrubbed:
info('No cleanup required')
+ # This has to be rerun to see if it has been successful
+ if lvm_snap_information():
+ return False
+ return True
+
+ return False
+
def do_client():
debug("Do client main loop")
should_cleanup = True
@@ -914,7 +938,7 @@
raise ClientException(line[5:].strip())
elif line.startswith('TRACEBACK'):
i = line.find('>>>')
- raise ClientException(line[10:i].strip(), line[i+3:].replace('###', '\n'))
+ raise ClientException(line[10:i].strip(), line[i+3:].replace('###', '\n').rstrip())
elif not line:
raise Exception('client died unexpectedly')
else:
@@ -1064,7 +1088,7 @@
try:
os.makedirs(datadir)
except EnvironmentError, ex:
- raise Exception('Can not create data store dir: %s' % datadir)
+ raise Exception('Can not create data store dir: %s: %s' % (datadir, ex))
rdiff_logdir = os.path.join(datadir, 'rdiff-backup-data')
if cfg['retention'] and os.path.isdir(rdiff_logdir) and not cleanup:
@@ -1098,8 +1122,6 @@
cin.flush()
do_server_getanswer(cout)
bdir = '/' # Fake directory for the rest of the cleanup
- do_server_rdiff_cleanup(cfg)
- cleaned_up = True
errs = 0
else:
cin.write('SETUP\n')
@@ -1121,6 +1143,7 @@
backup_marker = None
do_server_rdiff(cfg, bdir, nice, ionice, force)
+ cleaned_up = True
errs = 0
if os.path.isdir(rdiff_logdir):
@@ -1142,16 +1165,23 @@
info('Server backup for client %s: OK (%d WARNINGS)' % (id, errs))
except Exception, ex:
- if cleanup and not cleaned_up:
+ if cleanup:
info('Client-side cleanup for client %s: FAILED' % id)
- do_server_rdiff_cleanup(cfg)
else:
if isinstance(ex, ClientException):
error('Client %s: FAILED due to: %s' % (id, ex or ''))
- if ex.traceback: error(ex.traceback)
+ if ex.traceback and verbosity_level > 2: error(ex.traceback)
else:
error('Server backup for client %s: FAILED' % id, ex)
+ # Shutdown client
+ cout.close()
+ cin.close()
+
+ if not cleaned_up:
+ do_server_rdiff_cleanup(cfg)
+ cleaned_up = True
+
if output_done:
info('------------------------------------------------------------------')
debug('Server backup done')
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <fcr...@us...> - 2011-12-02 12:51:58
|
Revision: 776
http://safekeep.svn.sourceforge.net/safekeep/?rev=776&view=rev
Author: fcrawford
Date: 2011-12-02 12:51:50 +0000 (Fri, 02 Dec 2011)
Log Message:
-----------
Joe Steele <jo...@ma...>
Maintain compatibility with Python 2.2
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2011-11-28 12:38:37 UTC (rev 775)
+++ safekeep/trunk/safekeep 2011-12-02 12:51:50 UTC (rev 776)
@@ -19,7 +19,7 @@
import getopt, os, os.path, re, sys, fnmatch, stat, types
import commands, tempfile, time, traceback
import getpass, pwd, xml.dom.minidom
-import socket, smtplib
+import socket, smtplib, cStringIO
try:
import subprocess
@@ -128,7 +128,9 @@
return errs
def stacktrace():
- return "\n" + traceback.format_exc()
+ exc_file = cStringIO.StringIO()
+ traceback.print_exc(None, exc_file)
+ return "\n" + exc_file.getvalue()
def debug(msg):
log(msg, 'DBG')
@@ -171,7 +173,8 @@
if _stderr:
(child_in, child_out) = os.popen4(args)
else:
- (child_in, child_out) = os.popen3(args)
+ (child_in, child_out, child_err) = os.popen3(args)
+ child_err.close()
if not stdin:
child_in.close()
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <fcr...@us...> - 2011-12-03 02:36:08
|
Revision: 777
http://safekeep.svn.sourceforge.net/safekeep/?rev=777&view=rev
Author: fcrawford
Date: 2011-12-03 02:36:01 +0000 (Sat, 03 Dec 2011)
Log Message:
-----------
Futher updates to cleanup code for failed backups
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2011-12-02 12:51:50 UTC (rev 776)
+++ safekeep/trunk/safekeep 2011-12-03 02:36:01 UTC (rev 777)
@@ -649,13 +649,17 @@
lvmdev = device;
else:
lvmdev = '/dev/mapper/%s-%s' % (group, volume.replace('-', '--'))
- for i in range(1,10):
- ret = spawn(['sync'])
- ret = spawn(['dmsetup', 'remove', lvmdev])
- ret = spawn(['dmsetup', 'remove', lvmdev + '-cow'])
- ret = spawn(['lvremove', '--force', device])
- if not ret:
- break
+ if os.path.exists(lvmdev):
+ for i in range(1,10):
+ ret = spawn(['sync'])
+ ret = spawn(['dmsetup', 'remove', lvmdev])
+ ret = spawn(['dmsetup', 'remove', lvmdev + '-cow'])
+ ret = spawn(['lvremove', '--force', device])
+ if not ret:
+ break
+ else:
+ warn('lvremove called for non-existent device: %s' % lvmdev)
+ ret = 0 # Equivalent to lvremove succeeding
return ret
def gather_snap_information(device, bdir):
@@ -798,8 +802,7 @@
# Go through and see if any come from existing safekeep processes
pattern = re.compile(r"_snap_safekeep-(\d+)-")
- lvm_snap_list = lvm_snap_information()
- for (volume, group) in lvm_snap_list:
+ for (volume, group) in lvm_snap_information():
matches = pattern.search(volume)
if matches is not None:
pid = matches.group(1)
@@ -847,7 +850,7 @@
# Now cleanup any snapshots still hanging around
debug("Cleaning up remaining snapshots")
- for (volume, group) in lvm_snap_list:
+ for (volume, group) in lvm_snap_information():
device = os.path.join('/dev', group, volume)
info("Removing snapshot %s" % device)
ret = do_lvremove(device)
@@ -1078,7 +1081,7 @@
info('Server backup starting for client %s' % id)
output_done = True
- cleaned_up = False
+ cleaned_up = True
try:
if cfg['host']:
if not os.path.isfile(cfg['key_ctrl']):
@@ -1121,6 +1124,7 @@
cin.flush()
do_server_getanswer(cout)
if cleanup:
+ cleaned_up = False
cin.write('SCRUB\n')
cin.flush()
do_server_getanswer(cout)
@@ -1145,6 +1149,7 @@
else:
backup_marker = None
+ cleaned_up = False
do_server_rdiff(cfg, bdir, nice, ionice, force)
cleaned_up = True
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <fcr...@us...> - 2011-12-06 12:04:54
|
Revision: 778
http://safekeep.svn.sourceforge.net/safekeep/?rev=778&view=rev
Author: fcrawford
Date: 2011-12-06 12:04:48 +0000 (Tue, 06 Dec 2011)
Log Message:
-----------
Patch that will correct issues with performing
"mount --rbind / /mnt/safekeep...".
Instead of recursively binding the entire tree it only binds filesystems
that are mounted on "real" devices (i.e. have a device in the filesystem)
and that are not in the exclude list for the backup.
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2011-12-03 02:36:01 UTC (rev 777)
+++ safekeep/trunk/safekeep 2011-12-06 12:04:48 UTC (rev 778)
@@ -715,6 +715,33 @@
if ret:
warn('Can not tear down snapshot: %s' % device)
+def mount_excluded(cfg, mountpoint):
+ debug("mount_excluded: %s" % mountpoint)
+ mountpoint = mountpoint + '/'
+ for clude in cfg['cludes']:
+ if clude['type'] == 'exclude' and clude['path']:
+ if mountpoint.startswith(clude['path']):
+ debug("mount_excluded: %s: matched %s" % (mountpoint, clude['path']))
+ return True
+ debug("mount_excluded: %s: no matches" % mountpoint)
+ return False
+
+def do_rbind(cfg, startpath, bdir):
+ for (device, mountpoint, mounttype, mountoptions) in mount_information(False):
+ debug("Testing %s on %s" % (mountpoint, device))
+ if mountpoint.startswith(startpath) and device.startswith('/'):
+ if not mount_excluded(cfg, mountpoint):
+ ret = spawn(['mount', '--bind', mountpoint, reroot(bdir, mountpoint)])
+ if ret:
+ debug("mount --bind %s: failed: unwinding" % mountpoint)
+ ret = spawn(['umount', '-l', reroot(bdir, startpath)])
+ if ret:
+ warn('Failed to unmount: %s' % reroot(bdir, startpath))
+ return 1
+ spawn(['mount', '--make-unbindable', reroot(bdir, mountpoint)])
+
+ return 0
+
######################################################################
# Client implementation
######################################################################
@@ -753,7 +780,7 @@
if ret:
warn('modprobe dm-snapshot failed, continuing')
bdir = tempfile.mkdtemp("-rbind", "safekeep-%d-" % current_pid, "/mnt")
- ret = spawn(['mount', '--rbind', '/', bdir])
+ ret = do_rbind(cfg, '/', bdir)
if ret:
warn('mount --rbind failed, snapshotting will be disabled')
try:
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <fcr...@us...> - 2011-12-30 08:50:31
|
Revision: 783
http://safekeep.svn.sourceforge.net/safekeep/?rev=783&view=rev
Author: fcrawford
Date: 2011-12-30 08:50:25 +0000 (Fri, 30 Dec 2011)
Log Message:
-----------
Only add trailing slash on directory if it is not already present
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2011-12-30 08:43:51 UTC (rev 782)
+++ safekeep/trunk/safekeep 2011-12-30 08:50:25 UTC (rev 783)
@@ -724,7 +724,7 @@
def mount_excluded(cfg, mountpoint):
debug("mount_excluded: %s" % mountpoint)
- mountpoint = mountpoint + '/'
+ if not mountpoint.endswith('/'): mountpoint = mountpoint + '/'
for clude in cfg['cludes']:
if clude['type'] == 'exclude' and clude['path']:
if mountpoint.startswith(clude['path']):
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <fcr...@us...> - 2012-01-01 03:31:16
|
Revision: 786
http://safekeep.svn.sourceforge.net/safekeep/?rev=786&view=rev
Author: fcrawford
Date: 2012-01-01 03:31:10 +0000 (Sun, 01 Jan 2012)
Log Message:
-----------
Better reporting of server configuration errors, similar to other exception handling
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2012-01-01 03:28:15 UTC (rev 785)
+++ safekeep/trunk/safekeep 2012-01-01 03:31:10 UTC (rev 786)
@@ -1725,7 +1725,14 @@
base_dir = home_dir
if len(cfglocs) > 0:
- cfgs = parse_locs(cfglocs)
+ try:
+ cfgs = parse_locs(cfglocs)
+ except Exception, ex:
+ if isinstance(ex, ConfigException):
+ error('CONFIG ERROR: %s' % (ex or ''), ex)
+ else:
+ error('ERROR: %s' % (ex or ''), ex)
+ sys.exit(2)
else:
cfgs = {}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <fcr...@us...> - 2012-01-01 04:58:24
|
Revision: 787
http://safekeep.svn.sourceforge.net/safekeep/?rev=787&view=rev
Author: fcrawford
Date: 2012-01-01 04:58:17 +0000 (Sun, 01 Jan 2012)
Log Message:
-----------
Fix up verbosity_level to allow debug messages within configuration parsing
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2012-01-01 03:31:10 UTC (rev 786)
+++ safekeep/trunk/safekeep 2012-01-01 04:58:17 UTC (rev 787)
@@ -1556,6 +1556,8 @@
usage(2)
global backup_user, home_dir, base_dir
+ global verbosity_level
+
mode = None
email = []
email_from = None
@@ -1726,6 +1728,7 @@
if len(cfglocs) > 0:
try:
+ verbosity_level = 1 + verbosity
cfgs = parse_locs(cfglocs)
except Exception, ex:
if isinstance(ex, ConfigException):
@@ -1749,7 +1752,7 @@
if not ok: sys.exit(2)
try:
- global is_client, verbosity_level, verbosity_ssh, verbosity_trickle
+ global is_client, verbosity_ssh, verbosity_trickle
if verbosity > 2:
verbosity_trickle = verbosity_ssh = '-' + (verbosity-2) * 'v'
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <fcr...@us...> - 2012-01-02 03:17:51
|
Revision: 789
http://safekeep.svn.sourceforge.net/safekeep/?rev=789&view=rev
Author: fcrawford
Date: 2012-01-02 03:17:45 +0000 (Mon, 02 Jan 2012)
Log Message:
-----------
Corrected test for usable trickle as reported by Ken Bass
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2012-01-01 07:59:55 UTC (rev 788)
+++ safekeep/trunk/safekeep 2012-01-02 03:17:45 UTC (rev 789)
@@ -1119,7 +1119,7 @@
if limit_ul:
trickle.extend(['-u', str(limit_ul)])
if len(trickle):
- if try_to_run([trickle_cmd, '-V']) is not None:
+ if try_to_run([trickle_cmd, '-V']) is None:
warn('Trickle not available, bandwidth limiting disabled')
trickle = []
args.extend(trickle)
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <fcr...@us...> - 2012-01-13 11:47:22
|
Revision: 791
http://safekeep.svn.sourceforge.net/safekeep/?rev=791&view=rev
Author: fcrawford
Date: 2012-01-13 11:47:11 +0000 (Fri, 13 Jan 2012)
Log Message:
-----------
Clean up simple coding issues reported by pylint
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2012-01-08 00:01:59 UTC (rev 790)
+++ safekeep/trunk/safekeep 2012-01-13 11:47:11 UTC (rev 791)
@@ -66,7 +66,6 @@
base_dir = None
current_pid = os.getpid()
default_bandwidth = {}
-cmd = "<Missing>"
PROTOCOL = "1.2"
VERSION = "1.3.3"
@@ -110,10 +109,10 @@
else:
print >> sys.stderr, msg.encode('utf-8')
-def info_file(file, marker=None):
- info('# File: ' + file)
- errs = 0;
- fin = open(file, 'r')
+def info_file(filename, marker=None):
+ info('# File: ' + filename)
+ errs = 0
+ fin = open(filename, 'r')
try:
for line in fin.readlines():
if marker:
@@ -183,7 +182,7 @@
child_in.write(stdin)
child_in.close()
- lines=[]
+ lines = []
for line in child_out:
if stdout:
lines.append(line)
@@ -207,7 +206,7 @@
rc, out = do_spawn(args, stdin, stdout)
except OSError, ex:
ret = "OSError: %s" % (ex)
- error('%s failed: %s' % (cmd, ret));
+ error('%s failed: %s' % (cmd, ret))
return ret
if not rc:
@@ -219,7 +218,7 @@
else:
ret = 'unknown exit status: %d' % rc
if ret:
- error('%s failed: %s' % (cmd, ret));
+ error('%s failed: %s' % (cmd, ret))
return (ret, out)
# this just spawns an external program (optionally through a shell)
@@ -242,7 +241,7 @@
rc, out = do_spawn(args, None, True)
except OSError, ex:
return None
- if not rc in (0,1):
+ if not rc in (0, 1):
return None
return out or ''
@@ -265,8 +264,8 @@
cmd = ['/usr/sbin/sendmail', '-t', '-f', email_from]
call(cmd, stdin=msg)
-def is_temp_root(dir):
- return dir != '/'
+def is_temp_root(directory):
+ return directory != '/'
def reroot(root, path):
if root == '/': return path
@@ -275,9 +274,9 @@
if path.startswith('/'): return root + path
return os.path.join(root, path)
-def parse_prop_file(file):
+def parse_prop_file(filename):
props = {}
- fin = open(file)
+ fin = open(filename)
lines = fin.readlines()
fin.close()
for line in lines:
@@ -301,23 +300,23 @@
return repr(self.value)
def parse_dump(dump_el):
- type = dump_el.getAttribute('type')
- if not type:
+ dbtype = dump_el.getAttribute('type')
+ if not dbtype:
raise ConfigException('You need to specify the database type')
- if type not in ('postgres', 'postgresql', 'pgsql', 'mysql'):
- raise ConfigException('Invalid database type: %s' % type)
+ if dbtype not in ('postgres', 'postgresql', 'pgsql', 'mysql'):
+ raise ConfigException('Invalid database type: %s' % dbtype)
db = dump_el.getAttribute('db')
user = dump_el.getAttribute('user')
dbuser = dump_el.getAttribute('dbuser')
dbpasswd = dump_el.getAttribute('dbpasswd')
opts = (dump_el.getAttribute('options') or '').split()
- file = dump_el.getAttribute('file')
- if not file:
+ dbfile = dump_el.getAttribute('file')
+ if not dbfile:
raise ConfigException('You need to specify where the database should be dumped')
cleanup = dump_el.getAttribute('cleanup')
- return { 'type' : type, 'db' : db, 'user' : user, 'dbuser' : dbuser, 'dbpasswd': dbpasswd,
- 'opts' : opts, 'file' : file, 'cleanup' : cleanup }
+ return { 'type' : dbtype, 'db' : db, 'user' : user, 'dbuser' : dbuser, 'dbpasswd': dbpasswd,
+ 'opts' : opts, 'file' : dbfile, 'cleanup' : cleanup }
def parse_snap(snap_el):
device = snap_el.getAttribute('device')
@@ -358,8 +357,8 @@
def parse_config(backup_el, dflt_id):
if backup_el.tagName != 'backup':
raise ConfigException('Invalid config file, the top level element must be <backup>')
- id = backup_el.getAttribute('id')
- if not id: id = dflt_id
+ cfg_id = backup_el.getAttribute('id')
+ if not cfg_id: cfg_id = dflt_id
host_el = backup_el.getElementsByTagName('host')
if host_el:
@@ -392,15 +391,15 @@
raise ConfigException('Can not have more than one bandwidth element')
repo_el = backup_el.getElementsByTagName('repo')
- dir = None
+ repo_dir = None
retention = None
if len(repo_el) == 1:
- dir = repo_el[0].getAttribute('path')
+ repo_dir = repo_el[0].getAttribute('path')
retention = repo_el[0].getAttribute('retention')
elif len(repo_el) > 1:
raise ConfigException('Can not have more than one repo element')
- if not dir: dir = id
- dir = os.path.join(base_dir, dir)
+ if not repo_dir: repo_dir = cfg_id
+ repo_dir = os.path.join(base_dir, repo_dir)
options_els = backup_el.getElementsByTagName('options')
options = []
@@ -410,7 +409,7 @@
continue
option = options_el.nodeName
if option == 'special-files':
- warn('options element special-files is deprecated, use data attributes instead')
+ warn('options element special-files is deprecated, use data attributes instead')
if option in ('special-files', 'rdiff-backup'):
if options_el.hasAttributes():
for key, value in options_el.attributes.items():
@@ -466,8 +465,8 @@
'/var/named/chroot/var/run', '/var/named/chroot/var/tmp' ]
cludes = [{ 'type' : 'exclude', 'path' : path, 'glob' : None, 'regexp' : None } for path in path_xcludes]
- return { 'id': id, 'host' : host, 'port' : port, 'nice' : nice, 'user' : user, 'key_ctrl' : key_ctrl, 'key_data' : key_data,
- 'dir' : dir, 'retention' : retention, 'dumps' : dumps, 'snaps' : snaps, 'script' : script,
+ return { 'id': cfg_id, 'host' : host, 'port' : port, 'nice' : nice, 'user' : user, 'key_ctrl' : key_ctrl, 'key_data' : key_data,
+ 'dir' : repo_dir, 'retention' : retention, 'dumps' : dumps, 'snaps' : snaps, 'script' : script,
'cludes' : cludes, 'data_options' : data_options, 'options' : options, 'bw' : bw}
def parse_locs(cfglocs):
@@ -548,10 +547,10 @@
def do_client_dbdump(cfg):
debug('Doing DB dumps')
for dump in cfg['dumps']:
- type = dump['type']
+ dbtype = dump['type']
opts = dump['opts']
passwdfile = None
- if type in ('postgres', 'postgresql', 'pgsql'):
+ if dbtype in ('postgres', 'postgresql', 'pgsql'):
if dump['db']:
args = ['pg_dump']
args.extend(['-C'])
@@ -568,7 +567,7 @@
f.write(dump['dbpasswd'])
f.close()
- elif type in ('mysql'):
+ elif dbtype in ('mysql'):
args = ['mysqldump']
if dump['dbuser']:
args.extend(['-u', dump['dbuser']])
@@ -581,7 +580,7 @@
args.extend([dump['db']])
else:
- warn('Invalid database type: ' + type)
+ warn('Invalid database type: ' + dbtype)
continue
if dump['user']:
@@ -592,7 +591,7 @@
if passwdfile:
- os.environ['PGPASSFILE'] = passwdfile
+ os.environ['PGPASSFILE'] = passwdfile
try:
ec = spawn(cmd)
finally:
@@ -656,11 +655,11 @@
def do_lvremove(device):
(group, volume) = device.split('/')[-2:]
if group == 'mapper':
- lvmdev = device;
+ lvmdev = device
else:
lvmdev = '/dev/mapper/%s-%s' % (group, volume.replace('-', '--'))
if os.path.exists(lvmdev):
- for i in range(1,10):
+ for i in range(1, 10):
ret = spawn(['sync'])
ret = spawn(['dmsetup', 'remove', lvmdev])
ret = spawn(['dmsetup', 'remove', lvmdev + '-cow'])
@@ -1025,9 +1024,9 @@
client_side_script('POST-SETUP', cfg, bdir)
send('OK ' + bdir)
elif line.startswith('CLEANUP'):
- dir = line[7:].strip()
- if dir == bdir: should_cleanup = False
- do_client_cleanup(cfg, dir)
+ path = line[7:].strip()
+ if path == bdir: should_cleanup = False
+ do_client_cleanup(cfg, path)
client_side_script('POST-BACKUP', cfg, bdir)
send('OK')
elif line.startswith('SCRUB'):
@@ -1113,15 +1112,15 @@
limit_ul = get_bandwidth(cfg, 'upload')
if limit_dl or limit_ul:
trickle.extend([trickle_cmd])
- if verbosity_trickle: trickle.extend([verbosity_trickle])
+ if verbosity_trickle: trickle.extend([verbosity_trickle])
if limit_dl:
trickle.extend(['-d', str(limit_dl)])
if limit_ul:
trickle.extend(['-u', str(limit_ul)])
if len(trickle):
if try_to_run([trickle_cmd, '-V']) is None:
- warn('Trickle not available, bandwidth limiting disabled')
- trickle = []
+ warn('Trickle not available, bandwidth limiting disabled')
+ trickle = []
args.extend(trickle)
args.extend(['rdiff-backup'])
@@ -1139,7 +1138,7 @@
special_files = []
if cfg['data_options'].get('exclude-devices').lower() == 'true':
- special_files.extend(['--exclude-device-files'])
+ special_files.extend(['--exclude-device-files'])
if cfg['data_options'].get('exclude-sockets').lower() == 'true':
special_files.extend(['--exclude-sockets'])
if cfg['data_options'].get('exclude-fifos').lower() == 'true':
@@ -1203,10 +1202,10 @@
debug("Do server main loop")
output_done = False
for cfg in cfgs.itervalues():
- id = cfg['id']
- if ids and id not in ids: continue
+ cfg_id = cfg['id']
+ if ids and cfg_id not in ids: continue
info('------------------------------------------------------------------')
- info('Server backup starting for client %s' % id)
+ info('Server backup starting for client %s' % cfg_id)
output_done = True
cleaned_up = True
@@ -1231,9 +1230,9 @@
cmd = []
if cfg['host']:
cmd.extend(['ssh'])
- if verbosity_ssh: cmd.extend([verbosity_ssh])
+ if verbosity_ssh: cmd.extend([verbosity_ssh])
if cfg['port']: cmd.extend(['-p', cfg['port']])
- cmd.extend(['-T', '-i', cfg['key_ctrl'], '-l', cfg['user'], cfg['host']])
+ cmd.extend(['-T', '-i', cfg['key_ctrl'], '-l', cfg['user'], cfg['host']])
cmd.extend(['safekeep', '--client'])
if use_subprocess:
@@ -1248,7 +1247,7 @@
client_versions = do_server_getanswer(cout)
do_server_compat(client_versions)
- cin.write('CONFIG: %d: %s\n' % (len(cfg['text'].splitlines()), id))
+ cin.write('CONFIG: %d: %s\n' % (len(cfg['text'].splitlines()), cfg_id))
cin.write(cfg['text'] + '\n')
cin.flush()
remote_script = do_server_getanswer(cout)
@@ -1316,19 +1315,19 @@
do_server_getanswer(cout)
if errs == 0:
- info('Server backup for client %s: OK' % id)
+ info('Server backup for client %s: OK' % cfg_id)
else:
- info('Server backup for client %s: OK (%d WARNINGS)' % (id, errs))
+ info('Server backup for client %s: OK (%d WARNINGS)' % (cfg_id, errs))
except Exception, ex:
if cleanup:
- info('Client-side cleanup for client %s: FAILED' % id)
+ info('Client-side cleanup for client %s: FAILED' % cfg_id)
else:
- if isinstance(ex, ClientException):
- error('Client %s: FAILED due to: %s' % (id, ex or ''))
- if ex.traceback and verbosity_level > 2: error(ex.traceback)
- else:
- error('Server backup for client %s: FAILED' % id, ex)
+ if isinstance(ex, ClientException):
+ error('Client %s: FAILED due to: %s' % (cfg_id, ex or ''))
+ if ex.traceback and verbosity_level > 2: error(ex.traceback)
+ else:
+ error('Server backup for client %s: FAILED' % cfg_id, ex)
# Shutdown client
cout.close()
@@ -1346,13 +1345,13 @@
debug("Do server listing main loop")
output_done = False
for cfg in cfgs.itervalues():
- id = cfg['id']
- if ids and id not in ids: continue
+ cfg_id = cfg['id']
+ if ids and cfg_id not in ids: continue
if list_parsable:
- info('Client: %s' % id)
+ info('Client: %s' % cfg_id)
else:
info('------------------------------------------------------------------')
- info('Server listing for client %s' % id)
+ info('Server listing for client %s' % cfg_id)
output_done = True
args = ['rdiff-backup']
@@ -1382,11 +1381,11 @@
def do_keys(cfgs, ids, nice_rem, identity, status, dump, deploy):
for cfg in cfgs.itervalues():
- id = cfg['id']
- if ids and id not in ids: continue
- info('Handling keys for client: %s' % id)
+ cfg_id = cfg['id']
+ if ids and cfg_id not in ids: continue
+ info('Handling keys for client: %s' % cfg_id)
if not cfg['host']:
- info('%s: Client is local, it needs no keys' % id)
+ info('%s: Client is local, it needs no keys' % cfg_id)
continue
nice = cfg['nice'] or nice_rem
@@ -1403,25 +1402,25 @@
publickeyfile = privatekeyfile + '.pub'
if not os.path.isfile(privatekeyfile):
if os.path.isfile(publickeyfile):
- error('%s: Public key exists %s, but private key is missing. Skipping client.' % (id, publickeyfile))
+ error('%s: Public key exists %s, but private key is missing. Skipping client.' % (cfg_id, publickeyfile))
break
if dump:
- print '%s: Key does not exist: %s.' % (id, privatekeyfile)
+ print '%s: Key does not exist: %s.' % (cfg_id, privatekeyfile)
break
if status:
- print '%s: Key does not exist: %s. Will be generated.' % (id, privatekeyfile)
+ print '%s: Key does not exist: %s. Will be generated.' % (cfg_id, privatekeyfile)
break
if deploy:
- info('%s: Key do not exist, generating it now: %s' % (id, privatekeyfile))
+ info('%s: Key do not exist, generating it now: %s' % (cfg_id, privatekeyfile))
gencmd = 'ssh-keygen -q -b 1024 -t dsa -N "" -C "SafeKeep auto generated key at %s@%s" -f %s' % (backup_user, os.uname()[1], privatekeyfile)
if backup_user != work_user:
gencmd = 'su -s /bin/sh -c %s - %s' % (commands.mkarg(gencmd), backup_user)
debug(gencmd)
if spawn(gencmd):
- error('%s: Failed to generate key %s. Skipping client.' % (id, privatekeyfile))
+ error('%s: Failed to generate key %s. Skipping client.' % (cfg_id, privatekeyfile))
break
if not os.path.isfile(publickeyfile):
- error('%s: Private key exists %s, but public key is missing. Skipping client.' % (id, privatekeyfile))
+ error('%s: Private key exists %s, but public key is missing. Skipping client.' % (cfg_id, privatekeyfile))
break
fin = open(publickeyfile, 'r')
publickey = fin.read()
@@ -1445,7 +1444,7 @@
cmd = [basessh, '%s@%s' % (cfg['user'], cfg['host']), "if test -f .ssh/authorized_keys; then cat .ssh/authorized_keys; fi"]
authtext = call(cmd)
if authtext is None:
- error('%s: Failed to read the %s@%s:~/.ssh/authorized_keys file.' % (id, cfg['user'], cfg['host']))
+ error('%s: Failed to read the %s@%s:~/.ssh/authorized_keys file.' % (cfg_id, cfg['user'], cfg['host']))
continue
auth_keys = parse_authorized_keys(authtext)
this_keys = parse_authorized_keys(output)
@@ -1457,11 +1456,11 @@
new_keys.append(this_key)
if not new_keys:
if status:
- print '%s: Client is up to date.' % id
+ print '%s: Client is up to date.' % cfg_id
continue
if status:
- print '%s: Keys will be deployed on the client.' % id
+ print '%s: Keys will be deployed on the client.' % cfg_id
if deploy:
cmd = [basessh, '%s@%s' % (cfg['user'], cfg['host']), "umask 077; test -d .ssh || mkdir .ssh; cat >> .ssh/authorized_keys"]
keys = '%s\n' % '\n'.join([key[4] for key in new_keys])
@@ -1479,7 +1478,7 @@
if line[0] in '0123456789':
warn('SSH Protocol 1 keys are ignored: %s' % line)
continue
- opts =''
+ opts = ''
if line[0:7] not in ('ssh-dss', 'ssh-rsa'):
in_str = False
in_esc = False
@@ -1509,9 +1508,9 @@
error('Invalid key line, skipping: %s' % line)
continue
- type = parts[0]
- if type not in ('ssh-dss', 'ssh-rsa'):
- error('Invalid key type "%s", skipping: %s' % (type, line))
+ ssh_type = parts[0]
+ if ssh_type not in ('ssh-dss', 'ssh-rsa'):
+ error('Invalid key type "%s", skipping: %s' % (ssh_type, line))
continue
base46enc = parts[1]
@@ -1521,7 +1520,7 @@
else:
comment = parts[2]
- keys.append((opts, type, base46enc, comment, line))
+ keys.append((opts, ssh_type, base46enc, comment, line))
return keys
@@ -1697,7 +1696,7 @@
if cfgfile:
warn('Configuration file does not exist, skipping: %s' % cfgfile)
else:
- cfgfile = config_file
+ cfgfile = config_file
props = {}
def get_int(p):
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <fcr...@us...> - 2012-01-20 13:22:05
|
Revision: 793
http://safekeep.svn.sourceforge.net/safekeep/?rev=793&view=rev
Author: fcrawford
Date: 2012-01-20 13:21:54 +0000 (Fri, 20 Jan 2012)
Log Message:
-----------
Suppress duplicate message from client
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2012-01-20 12:25:15 UTC (rev 792)
+++ safekeep/trunk/safekeep 2012-01-20 13:21:54 UTC (rev 793)
@@ -334,7 +334,7 @@
if tag:
if not tag.startswith('@'): tag = '@' + tag.lstrip()
tags.append(tag.strip())
- else:
+ elif not is_client:
warn('Device: %s: empty tag in taglist: %s' % (device, tag_el))
return { 'device' : device, 'size' : size, 'tags' : tags }
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <fcr...@us...> - 2012-01-21 03:10:16
|
Revision: 794
http://safekeep.svn.sourceforge.net/safekeep/?rev=794&view=rev
Author: fcrawford
Date: 2012-01-21 03:10:10 +0000 (Sat, 21 Jan 2012)
Log Message:
-----------
Tie message to specific backup configuration
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2012-01-20 13:21:54 UTC (rev 793)
+++ safekeep/trunk/safekeep 2012-01-21 03:10:10 UTC (rev 794)
@@ -334,7 +334,7 @@
if tag:
if not tag.startswith('@'): tag = '@' + tag.lstrip()
tags.append(tag.strip())
- elif not is_client:
+ elif is_client:
warn('Device: %s: empty tag in taglist: %s' % (device, tag_el))
return { 'device' : device, 'size' : size, 'tags' : tags }
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <fcr...@us...> - 2012-02-11 05:39:00
|
Revision: 795
http://safekeep.svn.sourceforge.net/safekeep/?rev=795&view=rev
Author: fcrawford
Date: 2012-02-11 05:38:53 +0000 (Sat, 11 Feb 2012)
Log Message:
-----------
Added note about Python 2.2/2.3 support
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2012-01-21 03:10:10 UTC (rev 794)
+++ safekeep/trunk/safekeep 2012-02-11 05:38:53 UTC (rev 795)
@@ -34,8 +34,10 @@
# Python 2.2 compatibility
######################################################################
# There is no guarantee that we'll continue supporting Python 2.2
-# indefinitely, but we make a reasonable effor to do so as long as
+# indefinitely, but we make a reasonable effort to do so as long as
# it doesn't result in major complication/ugliness.
+# Note that some features related to scripting, snapshots and DB backups
+# already require Python 2.3.
try:
True, False
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <fcr...@us...> - 2012-02-11 06:07:21
|
Revision: 798
http://safekeep.svn.sourceforge.net/safekeep/?rev=798&view=rev
Author: fcrawford
Date: 2012-02-11 06:07:15 +0000 (Sat, 11 Feb 2012)
Log Message:
-----------
Update version for new release
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2012-02-11 06:05:29 UTC (rev 797)
+++ safekeep/trunk/safekeep 2012-02-11 06:07:15 UTC (rev 798)
@@ -70,7 +70,7 @@
default_bandwidth = {}
PROTOCOL = "1.2"
-VERSION = "1.3.3"
+VERSION = "1.4.0"
VEBOSITY_BY_CLASS = {'DBG': 3, 'INFO': 2, 'WARN': 1, 'ERR': 0}
######################################################################
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <fcr...@us...> - 2012-02-11 07:39:18
|
Revision: 799
http://safekeep.svn.sourceforge.net/safekeep/?rev=799&view=rev
Author: fcrawford
Date: 2012-02-11 07:39:12 +0000 (Sat, 11 Feb 2012)
Log Message:
-----------
Fixup up final testing failure
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2012-02-11 06:07:15 UTC (rev 798)
+++ safekeep/trunk/safekeep 2012-02-11 07:39:12 UTC (rev 799)
@@ -34,10 +34,8 @@
# Python 2.2 compatibility
######################################################################
# There is no guarantee that we'll continue supporting Python 2.2
-# indefinitely, but we make a reasonable effort to do so as long as
+# indefinitely, but we make a reasonable effor to do so as long as
# it doesn't result in major complication/ugliness.
-# Note that some features related to scripting, snapshots and DB backups
-# already require Python 2.3.
try:
True, False
@@ -995,26 +993,29 @@
send('OK %s, %s' % (PROTOCOL, VERSION))
elif line.startswith('CONFIG'):
cfg = do_client_config(line)
- if ':' in cfg['script']:
- (script_loc, script) = cfg['script'].split(':', 1)
- else:
- (script_loc, script) = ('client', cfg['script'])
- if script_loc == 'server':
- if not script_dir:
- script_dir = tempfile.mkdtemp(prefix="safekeep-%d-" % current_pid)
- script = os.path.basename(script)
- (fd, cfg['script']) = tempfile.mkstemp(prefix="%s-" % script, dir=script_dir)
- script_file = os.fdopen(fd, 'w')
- send('OK %s' % cfg['script'])
- elif script_loc == 'client':
- cfg['script'] = script
- ret = client_side_script('STARTUP', cfg, bdir)
- if ret:
- send('ERROR Client-side setup script failed: %s' % ret)
+ if cfg['script']:
+ if ':' in cfg['script']:
+ (script_loc, script) = cfg['script'].split(':', 1)
else:
+ (script_loc, script) = ('client', cfg['script'])
+ if script_loc == 'server':
+ if not script_dir:
+ script_dir = tempfile.mkdtemp(prefix="safekeep-%d-" % current_pid)
+ script = os.path.basename(script)
+ (fd, cfg['script']) = tempfile.mkstemp(prefix="%s-" % script, dir=script_dir)
+ script_file = os.fdopen(fd, 'w')
+ send('OK %s' % cfg['script'])
+ elif script_loc == 'client':
+ cfg['script'] = script
+ ret = client_side_script('STARTUP', cfg, bdir)
+ if ret:
+ send('ERROR Client-side setup script failed: %s' % ret)
+ else:
+ send('OK')
+ else:
+ warn('Unknown script location %s for script %s' % (script_loc, script))
send('OK')
else:
- warn('Unknown script location %s for script %s' % (script_loc, script))
send('OK')
elif line.startswith('SCRIPT'):
if do_client_remote_script(script_file, cfg, line):
@@ -1259,7 +1260,7 @@
cin.write(cfg['text'] + '\n')
cin.flush()
remote_script = do_server_getanswer(cout)
- if cfg['script'].startswith('server:') and remote_script:
+ if cfg['script'] and cfg['script'].startswith('server:') and remote_script:
local_script = cfg['script'].split(':', 1)[1]
if os.path.isfile(local_script):
ret = check_script_permissions(local_script)
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <fcr...@us...> - 2012-02-11 07:51:19
|
Revision: 800
http://safekeep.svn.sourceforge.net/safekeep/?rev=800&view=rev
Author: fcrawford
Date: 2012-02-11 07:51:13 +0000 (Sat, 11 Feb 2012)
Log Message:
-----------
Readded note about Python 2.2/2.3 support
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2012-02-11 07:39:12 UTC (rev 799)
+++ safekeep/trunk/safekeep 2012-02-11 07:51:13 UTC (rev 800)
@@ -34,8 +34,10 @@
# Python 2.2 compatibility
######################################################################
# There is no guarantee that we'll continue supporting Python 2.2
-# indefinitely, but we make a reasonable effor to do so as long as
+# indefinitely, but we make a reasonable effort to do so as long as
# it doesn't result in major complication/ugliness.
+# Note that some features related to scripting, snapshots and DB backups
+# already require Python 2.3.
try:
True, False
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <di...@us...> - 2012-02-14 20:40:13
|
Revision: 805
http://safekeep.svn.sourceforge.net/safekeep/?rev=805&view=rev
Author: dimi
Date: 2012-02-14 20:40:07 +0000 (Tue, 14 Feb 2012)
Log Message:
-----------
Oops, this got checked in by mistake, revert
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2012-02-14 20:38:19 UTC (rev 804)
+++ safekeep/trunk/safekeep 2012-02-14 20:40:07 UTC (rev 805)
@@ -475,9 +475,6 @@
'/var/named/chroot/var/run', '/var/named/chroot/var/tmp' ]
cludes = [{ 'type' : 'exclude', 'path' : path, 'glob' : None, 'regexp' : None } for path in path_xcludes]
- for dump in dumps:
- cludes.insert(0, { 'type' : 'include', 'path' : dump['file'], 'glob' : None, 'regexp' : None })
-
return { 'id': cfg_id, 'host' : host, 'port' : port, 'nice' : nice, 'user' : user, 'key_ctrl' : key_ctrl, 'key_data' : key_data,
'dir' : repo_dir, 'retention' : retention, 'dumps' : dumps, 'snaps' : snaps, 'script' : script,
'cludes' : cludes, 'data_options' : data_options, 'options' : options, 'bw' : bw}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <fcr...@us...> - 2012-04-06 13:21:31
|
Revision: 818
http://safekeep.svn.sourceforge.net/safekeep/?rev=818&view=rev
Author: fcrawford
Date: 2012-04-06 13:21:23 +0000 (Fri, 06 Apr 2012)
Log Message:
-----------
- generate a working "password file" for PostgreSQL by
Marco Bozzolan <ma...@s1...>
- set "nouuid" mount option for snapshots based on XFS filesystems from code by
Robert Jordan <rj...@no...> and
Alexander 'Leo' Bergolth <le...@st...>
- added attribute to supply mount options for snapshots as suggested by
Bryan Talbot <bt...@ae...>
- cleaned up coding for tempfile
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2012-02-18 13:26:47 UTC (rev 817)
+++ safekeep/trunk/safekeep 2012-04-06 13:21:23 UTC (rev 818)
@@ -68,6 +68,7 @@
base_dir = None
current_pid = os.getpid()
default_bandwidth = {}
+default_mountoptions = { 'xfs' : 'nouuid' }
PROTOCOL = "1.2"
VERSION = "1.4.0"
@@ -196,7 +197,6 @@
return (proc.wait(), lines)
else:
return (0, lines)
-
def _spawn(args, stdin=None, stdout=False):
if isinstance(args, types.StringTypes):
@@ -338,7 +338,8 @@
tags.append(tag.strip())
elif is_client:
warn('Device: %s: empty tag in taglist: %s' % (device, tag_el))
- return { 'device' : device, 'size' : size, 'tags' : tags }
+ mountoptions = snap_el.getAttribute('mount-options')
+ return { 'device' : device, 'size' : size, 'tags' : tags, 'mountoptions' : mountoptions }
def parse_clude(clude_el):
path = clude_el.getAttribute('path')
@@ -474,7 +475,7 @@
'/var/named/chroot/dev', '/var/named/chroot/proc',
'/var/named/chroot/var/run', '/var/named/chroot/var/tmp' ]
cludes = [{ 'type' : 'exclude', 'path' : path, 'glob' : None, 'regexp' : None } for path in path_xcludes]
-
+
return { 'id': cfg_id, 'host' : host, 'port' : port, 'nice' : nice, 'user' : user, 'key_ctrl' : key_ctrl, 'key_data' : key_data,
'dir' : repo_dir, 'retention' : retention, 'dumps' : dumps, 'snaps' : snaps, 'script' : script,
'cludes' : cludes, 'data_options' : data_options, 'options' : options, 'bw' : bw}
@@ -574,7 +575,7 @@
if dump['dbpasswd']:
(fd, passwdfile) = tempfile.mkstemp()
f = os.fdopen(fd, 'w')
- f.write(dump['dbpasswd'])
+ f.write('*:*:*:*:%s' % dump['dbpasswd'])
f.close()
elif dbtype in ('mysql'):
@@ -599,7 +600,6 @@
cmd = ' '.join([commands.mkarg(arg) for arg in args])
cmd = '%s > %s' % (cmd, commands.mkarg(dump['file']))
-
if passwdfile:
os.environ['PGPASSFILE'] = passwdfile
try:
@@ -711,7 +711,12 @@
warn('Can not snapshot the device: %s' % device)
continue
# no need to mkdir since the mountpoint already exists
- args = ['mount', '-t', snaptyp, snapdev, snapmnt]
+ args = ['mount', '-t', snaptyp]
+ if snap['mountoptions']:
+ args.extend(['-o', snap['mountoptions']])
+ elif snaptyp in default_mountoptions:
+ args.extend(['-o', default_mountoptions[snaptyp]])
+ args.extend([snapdev, snapmnt])
ec = spawn(args)
if ec:
warn('Can not mount the snapshot: %s' % device)
@@ -822,7 +827,7 @@
ret = spawn(['modprobe', 'dm-snapshot'])
if ret:
warn('modprobe dm-snapshot failed, continuing')
- bdir = tempfile.mkdtemp("-rbind", "safekeep-%d-" % current_pid, "/mnt")
+ bdir = tempfile.mkdtemp(suffix="-rbind", prefix="safekeep-%d-" % current_pid, dir="/mnt")
ret = do_rbind(cfg, '/', bdir)
if ret:
warn('mount --rbind failed, snapshotting will be disabled')
@@ -1003,6 +1008,7 @@
if script_loc == 'server':
if not script_dir:
script_dir = tempfile.mkdtemp(prefix="safekeep-%d-" % current_pid)
+ tempfile.tempdir = script_dir
script = os.path.basename(script)
(fd, cfg['script']) = tempfile.mkstemp(prefix="%s-" % script, dir=script_dir)
script_file = os.fdopen(fd, 'w')
@@ -1059,6 +1065,7 @@
if not script_file.closed: script_file.close()
os.remove(cfg['script'])
if script_dir:
+ tempfile.tempdir = None
try:
os.rmdir(script_dir)
except OSError, e:
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <fcr...@us...> - 2012-04-14 11:17:18
|
Revision: 820
http://safekeep.svn.sourceforge.net/safekeep/?rev=820&view=rev
Author: fcrawford
Date: 2012-04-14 11:17:12 +0000 (Sat, 14 Apr 2012)
Log Message:
-----------
Major rework of snapshot mounting
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2012-04-06 13:22:43 UTC (rev 819)
+++ safekeep/trunk/safekeep 2012-04-14 11:17:12 UTC (rev 820)
@@ -68,7 +68,10 @@
base_dir = None
current_pid = os.getpid()
default_bandwidth = {}
-default_mountoptions = { 'xfs' : 'nouuid' }
+# Default mount options, overridden elsewhere:
+# Key is a file system type, or 'snapshot' for default for snapshot mount
+# or 'bind' for a bind mount (check mount for details)
+default_mountoptions = { 'xfs' : 'ro,nouuid', 'snapshot' : 'ro', 'bind' : 'ro' }
PROTOCOL = "1.2"
VERSION = "1.4.0"
@@ -644,9 +647,11 @@
mounts.append(matches.groups())
return mounts
+def normalise_lvm_device(device):
+ return device.replace('/mapper','').replace('-','/').replace('//', '-')
+
def map_lvm_device(device):
- device = device.replace('/mapper','').replace('-','/').replace('//', '-')
- return device.split('/')[-2:]
+ return normalise_lvm_device(device).split('/')[-2:]
def check_lvm_information(device):
(group, volume) = map_lvm_device(device)
@@ -655,13 +660,6 @@
return True
return False
-def gather_lvm_information(device):
- (group, volume) = map_lvm_device(device)
- for (device, mountpoint, mounttype, mountoptions) in mount_information(False):
- if [group, volume] == map_lvm_device(device):
- return (group, volume, mountpoint, mounttype)
- return (None, None, None, None)
-
def do_lvremove(device):
(group, volume) = device.split('/')[-2:]
if group == 'mapper':
@@ -681,9 +679,9 @@
ret = 0 # Equivalent to lvremove succeeding
return ret
-def gather_snap_information(device, bdir):
- (group, volume, mountpoint, mounttype) = gather_lvm_information(device)
- if not mountpoint: return (None, None, None, None)
+def generate_snap_information(device, bdir, mountpoint, mounttype):
+ assert os.path.ismount(mountpoint)
+ (group, volume) = map_lvm_device(device)
lvmdev = os.path.join('/dev', group, volume)
if bdir[-1] == '/': bdir = bdir[:-1]
snapname = '%s_snap_%s' % (volume, os.path.basename(bdir))
@@ -691,58 +689,77 @@
if os.path.isabs(mountpoint[0]): mountpoint = mountpoint[1:]
return (lvmdev, snapdev, os.path.join(bdir, mountpoint), mounttype)
-def do_client_snap(cfg, bdir):
+def do_client_snap_device(snap, bdir, mountpoint, mounttype):
assert is_temp_root(bdir)
- debug('Doing FS snapshots')
- for snap in cfg['snaps']:
- device = snap['device']
- (lvmdev, snapdev, snapmnt, snaptyp) = gather_snap_information(device, bdir)
- if not snapmnt:
- warn('Cannot find the mountpoint for: %s' % device)
- continue
- args = ['lvcreate']
- if snap['tags']:
- for tag in snap['tags']:
- args.extend(['--addtag', tag.strip()])
- args.extend(['--snapshot', '--size', snap['size'],
- '--name', os.path.basename(snapdev), lvmdev])
- ec = spawn(args)
- if ec:
- warn('Can not snapshot the device: %s' % device)
- continue
- # no need to mkdir since the mountpoint already exists
- args = ['mount', '-t', snaptyp]
- if snap['mountoptions']:
- args.extend(['-o', snap['mountoptions']])
- elif snaptyp in default_mountoptions:
- args.extend(['-o', default_mountoptions[snaptyp]])
- args.extend([snapdev, snapmnt])
- ec = spawn(args)
- if ec:
- warn('Can not mount the snapshot: %s' % device)
- ret = do_lvremove(snapdev)
- if ret:
- warn('Can not tear down snapshot: %s' % device)
+ device = snap['device']
+ debug('Doing FS snapshot for %s' % device)
+ (lvmdev, snapdev, snapmnt, snaptyp) = generate_snap_information(device, bdir, mountpoint, mounttype)
+ if not os.path.isdir(snapmnt):
+ warn('Cannot find the mountpoint %s for %s' % (snapmnt, device))
+ return False
+ if 'snapdevice' in snap:
+ warn('bind mount of snapshot not currently supported: %s' % mountpoint)
+ return True # Should be False, but this will still do a good backup
+ args = ['lvcreate']
+ if snap['tags']:
+ for tag in snap['tags']:
+ args.extend(['--addtag', tag.strip()])
+ args.extend(['--snapshot', '--size', snap['size'],
+ '--name', os.path.basename(snapdev), lvmdev])
+ ec = spawn(args)
+ if ec:
+ warn('Can not snapshot the device: %s' % device)
+ return False
+ # no need to mkdir since the mountpoint already exists
+ args = ['mount', '-t', snaptyp]
+ if snap['mountoptions']:
+ args.extend(['-o', snap['mountoptions']])
+ elif snaptyp in default_mountoptions:
+ args.extend(['-o', default_mountoptions[snaptyp]])
+ elif 'snapshot' in default_mountoptions:
+ args.extend(['-o', default_mountoptions['snapshot']])
+ args.extend([snapdev, snapmnt])
+ ec = spawn(args)
+ if ec:
+ warn('Can not mount the snapshot: %s' % device)
+ ret = do_lvremove(snapdev)
+ if ret:
+ warn('Can not tear down snapshot: %s' % device)
+ return False
+ snap['snapdevice'] = snapdev
+ snap['mountpoint'] = snapmnt
+ return True
+
def do_client_snap_teardown(cfg, bdir):
assert is_temp_root(bdir)
debug('Tear down FS snapshots dumps')
snaps = list(cfg['snaps'])
snaps.reverse()
for snap in snaps:
+ if 'mountpoint' in snap: del snap['mountpoint']
device = snap['device']
- (lvmdev, snapdev, snapmnt, snaptyp) = gather_snap_information(device, bdir)
- if not snapmnt:
- warn('Can not find the mountpoint for: %s' % device)
+ if not 'snapdevice' in snap:
+ warn('No snapdevice for device teardown: %s' % device)
continue
- ret = spawn(['umount', snapmnt])
- if ret:
- warn('Can not umount the snapshot: %s' % snapmnt)
+ snapdev = snap['snapdevice']
+ debug('Tear down FS snapshot dump for %s -> %s' % (snapdev, device))
+
ret = do_lvremove(snapdev)
if ret:
warn('Can not tear down snapshot: %s' % device)
+ continue
+ del snap['snapdevice']
+def find_snapshot(cfg, device):
+ for snap in cfg['snaps']:
+ if normalise_lvm_device(snap['device']) == normalise_lvm_device(device):
+ debug('find_snapshot device matched: %s' % device)
+ return snap
+ debug('find_snapshot device no matched for %s' % device)
+ return None
+
def mount_excluded(cfg, mountpoint):
debug("mount_excluded: %s" % mountpoint)
if not mountpoint.endswith('/'): mountpoint = mountpoint + '/'
@@ -754,19 +771,42 @@
debug("mount_excluded: %s: no matches" % mountpoint)
return False
+def do_umount_all(bdir):
+ assert is_temp_root(bdir)
+ total_ret = 0
+ for (device, mountpoint, mounttype, mountoptions) in mount_information(True):
+ if mountpoint.startswith(bdir):
+ debug("Removing mount %s" % mountpoint)
+ ret = spawn(['umount', mountpoint])
+ if ret:
+ warn('Can not unmount snapshot: %s' % mountpoint)
+ total_ret += ret
+ return total_ret
+
def do_rbind(cfg, startpath, bdir):
for (device, mountpoint, mounttype, mountoptions) in mount_information(False):
debug("Testing %s on %s" % (mountpoint, device))
if mountpoint.startswith(startpath) and device.startswith('/'):
if not mount_excluded(cfg, mountpoint):
- ret = spawn(['mount', '--bind', mountpoint, reroot(bdir, mountpoint)])
+ snap = find_snapshot(cfg, device)
+ if snap:
+ ret = not do_client_snap_device(snap, bdir, mountpoint, mounttype)
+ else:
+ ret = spawn(['mount', '--bind', mountpoint, reroot(bdir, mountpoint)])
+ if ret:
+ debug("mount --bind %s: failed: unwinding" % mountpoint)
+ else:
+ spawn(['mount', '--make-unbindable', reroot(bdir, mountpoint)])
+ if 'bind' in default_mountoptions:
+ spawn(['mount' , '-o',
+ ('remount,%s,bind' % default_mountoptions['bind']),
+ mountpoint, reroot(bdir, mountpoint)])
if ret:
- debug("mount --bind %s: failed: unwinding" % mountpoint)
- ret = spawn(['umount', '-l', reroot(bdir, startpath)])
+ ret = spawn(['umount', reroot(bdir, startpath)])
if ret:
warn('Failed to unmount: %s' % reroot(bdir, startpath))
+ do_client_snap_teardown(cfg, bdir)
return 1
- spawn(['mount', '--make-unbindable', reroot(bdir, mountpoint)])
return 0
@@ -822,7 +862,7 @@
for snap in cfg['snaps']:
device = snap['device']
if check_lvm_information(device) and not do_client_scrub():
- raise Exception("Previous snapshots found for %s and automatic correction failed: run 'safekeep --server --cleanup' to correct" % device)
+ raise Exception("Previous snapshots found for %s and automatic correction failed: run 'safekeep --server --cleanup %s' to correct" % (device, cfg['host']))
ret = spawn(['modprobe', 'dm-snapshot'])
if ret:
@@ -836,8 +876,6 @@
except OSError, e:
warn('Failed to remove: %s: %s' % (bdir, e))
bdir = '/'
- else:
- do_client_snap(cfg, bdir)
else:
bdir = '/'
debug('Working root is %s' % bdir)
@@ -847,17 +885,17 @@
def do_client_cleanup(cfg, bdir):
debug('Do cleanup of %s in %s' % (cfg['host'], bdir))
if is_temp_root(bdir):
- do_client_snap_teardown(cfg, bdir)
-
- ret = spawn(['umount', '-l', bdir])
+ ret = do_umount_all(bdir)
if ret:
- warn('Failed to unmount: %s' % bdir)
+ warn('Failed to unmount tree: %s' % bdir)
else:
try:
os.rmdir(bdir)
except OSError, e:
warn('Unable to remove: %s: %s' % (bdir, e))
+ do_client_snap_teardown(cfg, bdir)
+
do_client_dbdump_teardown(cfg)
def do_client_compat(server_versions):
@@ -903,9 +941,9 @@
info("Removing mount %s" % mountpoint)
if device == '/' and 'bind' in mountoptions.split(','):
info("Removing rbind directory %s" % mountpoint)
- ret = spawn(['umount', '-l', mountpoint])
+ ret = do_umount_all(mountpoint)
if ret:
- warn('Failed to unmount: %s' % mountpoint)
+ warn('Failed to unmount tree: %s' % mountpoint)
else:
try:
os.rmdir(mountpoint)
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <fcr...@us...> - 2012-04-22 09:13:28
|
Revision: 822
http://safekeep.svn.sourceforge.net/safekeep/?rev=822&view=rev
Author: fcrawford
Date: 2012-04-22 09:13:22 +0000 (Sun, 22 Apr 2012)
Log Message:
-----------
Patch for bind mount issue as reported by Andres Toomsalu <an...@ac...>
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2012-04-15 16:01:23 UTC (rev 821)
+++ safekeep/trunk/safekeep 2012-04-22 09:13:22 UTC (rev 822)
@@ -788,6 +788,9 @@
debug("Testing %s on %s" % (mountpoint, device))
if mountpoint.startswith(startpath) and device.startswith('/'):
if not mount_excluded(cfg, mountpoint):
+ if 'bind' in mountoptions.split(','):
+ warn('bind mount of snapshot not currently supported: %s' % mountpoint)
+ continue
snap = find_snapshot(cfg, device)
if snap:
ret = not do_client_snap_device(snap, bdir, mountpoint, mounttype)
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <fcr...@us...> - 2012-05-04 13:06:29
|
Revision: 823
http://safekeep.svn.sourceforge.net/safekeep/?rev=823&view=rev
Author: fcrawford
Date: 2012-05-04 13:06:22 +0000 (Fri, 04 May 2012)
Log Message:
-----------
Add writable options for snapshot and bind mounts
Modified Paths:
--------------
safekeep/trunk/safekeep
Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep 2012-04-22 09:13:22 UTC (rev 822)
+++ safekeep/trunk/safekeep 2012-05-04 13:06:22 UTC (rev 823)
@@ -304,6 +304,14 @@
def __str__(self):
return repr(self.value)
+def parse_true_false(el, attr, default = None):
+ true_false = el.getAttribute(attr).lower()
+ if true_false and not true_false in ('true', 'false'):
+ raise ConfigException('Option needs to be true or false: attr %s: value %s' % (attr, el.getAttribute(attr)))
+ if not true_false:
+ true_false = default
+ return true_false
+
def parse_dump(dump_el):
dbtype = dump_el.getAttribute('type')
if not dbtype:
@@ -319,7 +327,7 @@
dbfile = dump_el.getAttribute('file')
if not dbfile:
raise ConfigException('You need to specify where the database should be dumped')
- cleanup = dump_el.getAttribute('cleanup')
+ cleanup = parse_true_false(dump_el, 'cleanup')
return { 'type' : dbtype, 'db' : db, 'user' : user, 'dbuser' : dbuser, 'dbpasswd': dbpasswd,
'opts' : opts, 'file' : dbfile, 'cleanup' : cleanup }
@@ -342,7 +350,8 @@
elif is_client:
warn('Device: %s: empty tag in taglist: %s' % (device, tag_el))
mountoptions = snap_el.getAttribute('mount-options')
- return { 'device' : device, 'size' : size, 'tags' : tags, 'mountoptions' : mountoptions }
+ writable = parse_true_false(snap_el, 'writable')
+ return { 'device' : device, 'size' : size, 'tags' : tags, 'mountoptions' : mountoptions, 'snap_writable' : writable }
def parse_clude(clude_el):
path = clude_el.getAttribute('path')
@@ -363,9 +372,9 @@
def parse_data_attributes(data_el):
return {
- 'exclude-devices': (data_el.getAttribute('exclude-devices') or 'false'),
- 'exclude-sockets': (data_el.getAttribute('exclude-sockets') or 'false'),
- 'exclude-fifos': (data_el.getAttribute('exclude-fifos') or 'false')
+ 'exclude-devices': parse_true_false(data_el, 'exclude-devices', 'false'),
+ 'exclude-sockets': parse_true_false(data_el, 'exclude-sockets', 'false'),
+ 'exclude-fifos': parse_true_false(data_el, 'exclude-fifos', 'false')
}
def parse_config(backup_el, dflt_id):
@@ -434,10 +443,12 @@
raise ConfigException('Unknown option "%s"' % option)
setup_el = backup_el.getElementsByTagName('setup')
+ writable = None
dumps = []
snaps = []
script = None
if len(setup_el) > 0:
+ writable = parse_true_false(setup_el[0], 'writable')
dump_els = setup_el[0].getElementsByTagName('dump')
for dump_el in dump_els:
dumps.append(parse_dump(dump_el))
@@ -480,7 +491,7 @@
cludes = [{ 'type' : 'exclude', 'path' : path, 'glob' : None, 'regexp' : None } for path in path_xcludes]
return { 'id': cfg_id, 'host' : host, 'port' : port, 'nice' : nice, 'user' : user, 'key_ctrl' : key_ctrl, 'key_data' : key_data,
- 'dir' : repo_dir, 'retention' : retention, 'dumps' : dumps, 'snaps' : snaps, 'script' : script,
+ 'dir' : repo_dir, 'retention' : retention, 'dumps' : dumps, 'snaps' : snaps, 'script' : script, 'mount_writable' : writable,
'cludes' : cludes, 'data_options' : data_options, 'options' : options, 'bw' : bw}
def parse_locs(cfglocs):
@@ -617,7 +628,7 @@
def do_client_dbdump_teardown(cfg):
debug('Tear down DB dumps')
for dump in cfg['dumps']:
- if dump['cleanup'].lower() != 'true':
+ if dump['cleanup'] != 'true':
continue
try:
os.remove(dump['file'])
@@ -689,7 +700,21 @@
if os.path.isabs(mountpoint[0]): mountpoint = mountpoint[1:]
return (lvmdev, snapdev, os.path.join(bdir, mountpoint), mounttype)
-def do_client_snap_device(snap, bdir, mountpoint, mounttype):
+def update_mountoptions(mountoptions, writable):
+ if writable == 'true':
+ sub = 'rw'
+ else:
+ sub = 'ro'
+ mod = False
+ options = mountoptions.split(',')
+ for i in range(len(options)):
+ if options[i] in ('rw', 'ro'):
+ options[i] = sub
+ mod = True
+ if not mod: options.append(sub)
+ return ','.join(options)
+
+def do_client_snap_device(snap, bdir, mountpoint, mounttype, writable):
assert is_temp_root(bdir)
device = snap['device']
debug('Doing FS snapshot for %s' % device)
@@ -714,11 +739,17 @@
# no need to mkdir since the mountpoint already exists
args = ['mount', '-t', snaptyp]
if snap['mountoptions']:
- args.extend(['-o', snap['mountoptions']])
+ mountoptions = snap['mountoptions']
elif snaptyp in default_mountoptions:
- args.extend(['-o', default_mountoptions[snaptyp]])
+ mountoptions = default_mountoptions[snaptyp]
elif 'snapshot' in default_mountoptions:
- args.extend(['-o', default_mountoptions['snapshot']])
+ mountoptions = default_mountoptions['snapshot']
+ if snap['snap_writable']:
+ writable = snap['snap_writable']
+ if writable:
+ mountoptions = update_mountoptions(mountoptions, writable)
+ if mountoptions:
+ args.extend(['-o', mountoptions])
args.extend([snapdev, snapmnt])
ec = spawn(args)
if ec:
@@ -793,7 +824,7 @@
continue
snap = find_snapshot(cfg, device)
if snap:
- ret = not do_client_snap_device(snap, bdir, mountpoint, mounttype)
+ ret = not do_client_snap_device(snap, bdir, mountpoint, mounttype, cfg['mount_writable'])
else:
ret = spawn(['mount', '--bind', mountpoint, reroot(bdir, mountpoint)])
if ret:
@@ -801,8 +832,12 @@
else:
spawn(['mount', '--make-unbindable', reroot(bdir, mountpoint)])
if 'bind' in default_mountoptions:
+ mountoptions = default_mountoptions['bind']
+ if cfg['mount_writable']:
+ mountoptions = update_mountoptions(mountoptions, cfg['mount_writable'])
+ if mountoptions:
spawn(['mount' , '-o',
- ('remount,%s,bind' % default_mountoptions['bind']),
+ ('remount,%s,bind' % mountoptions),
mountpoint, reroot(bdir, mountpoint)])
if ret:
ret = spawn(['umount', reroot(bdir, startpath)])
@@ -942,20 +977,9 @@
for (device, mountpoint, mounttype, mountoptions) in mount_information(True):
if mountpoint.startswith('/mnt/safekeep-'):
info("Removing mount %s" % mountpoint)
- if device == '/' and 'bind' in mountoptions.split(','):
- info("Removing rbind directory %s" % mountpoint)
- ret = do_umount_all(mountpoint)
- if ret:
- warn('Failed to unmount tree: %s' % mountpoint)
- else:
- try:
- os.rmdir(mountpoint)
- except OSError, e:
- warn('Failed to remove: %s: %s' % (mountpoint, e))
- else:
- ret = spawn(['umount', mountpoint])
- if ret:
- warn('Can not unmount the snapshot: %s' % mountpoint)
+ ret = spawn(['umount', mountpoint])
+ if ret:
+ warn('Can not unmount the snapshot: %s' % mountpoint)
if fnmatch.fnmatch(device, '*_snap_safekeep-*'):
info("Removing snapshot %s" % device)
ret = do_lvremove(device)
@@ -1196,11 +1220,11 @@
options_append = []
special_files = []
- if cfg['data_options'].get('exclude-devices').lower() == 'true':
+ if cfg['data_options'].get('exclude-devices') == 'true':
special_files.extend(['--exclude-device-files'])
- if cfg['data_options'].get('exclude-sockets').lower() == 'true':
+ if cfg['data_options'].get('exclude-sockets') == 'true':
special_files.extend(['--exclude-sockets'])
- if cfg['data_options'].get('exclude-fifos').lower() == 'true':
+ if cfg['data_options'].get('exclude-fifos') == 'true':
special_files.extend(['--exclude-fifos'])
for option in cfg['options']:
@@ -1740,7 +1764,7 @@
if mode != 'server' and (email or smtp):
usage(2)
- if not mode in ['server', 'client'] and cleanup:
+ if not mode in ('server', 'client') and cleanup:
usage(2)
if mode == 'client' and cfglocs:
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|