|
From: Will P. <pa...@us...> - 2006-02-07 15:30:06
|
Update of /cvsroot/ark/sidai/dchunk-utils In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv28042/dchunk-utils Modified Files: acct-for-diskspace sync-replica Log Message: updates Index: acct-for-diskspace =================================================================== RCS file: /cvsroot/ark/sidai/dchunk-utils/acct-for-diskspace,v retrieving revision 1.16 retrieving revision 1.17 diff -C2 -r1.16 -r1.17 *** acct-for-diskspace 4 Apr 2005 10:09:38 -0000 1.16 --- acct-for-diskspace 7 Feb 2006 11:40:21 -0000 1.17 *************** *** 71,75 **** """ ! import os,re,string,sys import ark.control import ark.dchunk --- 71,75 ---- """ ! import errno,os,re,string,sys import ark.control import ark.dchunk *************** *** 208,213 **** (device,mountpt,mttype) = match.group(1,2,3) ! # print l, ! # print '<',device,'><',mountpt, '><',mttype, '>' # we're only interested in real, live disk mounts, --- 208,213 ---- (device,mountpt,mttype) = match.group(1,2,3) ! #print l, ! #print '<',device,'><',mountpt, '><',mttype, '>' # we're only interested in real, live disk mounts, *************** *** 261,265 **** def drillDown(level,host,dir): hostname = host._name ! #sys.stderr.write('lev%d: %s:%s\n'%(level,hostname,dir)) if ACCTD.has_key(hostname) and ACCTD[hostname].has_key(dir): ACCTD[hostname][dir] = 2 # done --- 261,265 ---- def drillDown(level,host,dir): hostname = host._name ! # sys.stderr.write('lev%d: %s:%s\n'%(level,hostname,dir)) if ACCTD.has_key(hostname) and ACCTD[hostname].has_key(dir): ACCTD[hostname][dir] = 2 # done *************** *** 273,276 **** --- 273,286 ---- print 'f %s:%s' % (hostname,dir) return + except OSError,e: + if e.errno == errno.EACCES: + print 'P %s:%s # permissions...' % (hostname,dir) + return + else: + # print e.errno + raise # again + except: + # print `e` + raise if not in_play: *************** *** 283,287 **** for sd in substuff: ! dsd = '%s/%s' % (dir,sd) if IGNORE_DIRS.has_key(sd): continue # specifically uninteresting if not in_play: --- 293,300 ---- for sd in substuff: ! if dir == '/': ! dsd = '/%s' % (sd) ! else: ! dsd = '%s/%s' % (dir,sd) if IGNORE_DIRS.has_key(sd): continue # specifically uninteresting if not in_play: Index: sync-replica =================================================================== RCS file: /cvsroot/ark/sidai/dchunk-utils/sync-replica,v retrieving revision 1.10 retrieving revision 1.11 diff -C2 -r1.10 -r1.11 *** sync-replica 30 Jul 2005 12:35:33 -0000 1.10 --- sync-replica 7 Feb 2006 11:40:21 -0000 1.11 *************** *** 2,6 **** ################################################################ # ! # Copyright (c) 2002-2004, Will Partain # All rights reserved. # --- 2,6 ---- ################################################################ # ! # Copyright (c) 2002-2005, Will Partain # All rights reserved. # *************** *** 55,67 **** --help Print this message and exit. --rdiff-backup Use rdiff-backup instead of rsync. --rsyncer-verbose Run rsync/rdiff-backup verbosely. """ ! import os,re,sys #import os,re,string,sys #import ark.control import ark.dchunk import ark.host #import ark.error --- 55,70 ---- --help Print this message and exit. --rdiff-backup Use rdiff-backup instead of rsync. + --dated-snapshots Do dated snapshots with rsync (instead + of plain rsync or rdiff-backup) --rsyncer-verbose Run rsync/rdiff-backup verbosely. """ ! import datetime,os,re,string,sys #import os,re,string,sys #import ark.control import ark.dchunk import ark.host + import ark.utils # globNormal #import ark.error *************** *** 77,80 **** --- 80,84 ---- , 'pretend' , 'rdiff-backup' + , 'dated-snapshots' , 'rsyncer-verbose' ] *************** *** 84,88 **** if opts.isSet('quiet'): verbose = None pretend = opts.isSet('pretend') ! use_rdiff_backup = opts.isSet('rdiff-backup') non_opts = opts.cmdLineNonOptions() --- 88,97 ---- if opts.isSet('quiet'): verbose = None pretend = opts.isSet('pretend') ! use_rdiff_backup = opts.isSet('rdiff-backup') ! use_dated_snapshots = opts.isSet('dated-snapshots') ! ! if use_rdiff_backup and use_dated_snapshots: ! sys.stderr.write('Only one of: --rdiff-backup, --dated-snapshots\n') ! sys.exit(1) non_opts = opts.cmdLineNonOptions() *************** *** 117,120 **** --- 126,156 ---- (dest_hostname,dest_dir,nfs4_junk_inFo) = replica.masterHostnameAndDir() + if use_dated_snapshots: + prior_dir = dest_dir # remember this root... + prior_dir_stuff = '' + today = datetime.date.today().isoformat() # YYYY-MM-DD + dest_dir += '/%s' % (today) + + cands = ark.utils.globNormal(prior_dir,host_spec='.',globb='????-??-??') + #print >>sys.stderr, 'cands=',cands + + # there could be zero to many candidates; we want --link-dest args + # for (up to) the last five [arbitrary] candidates. Not one, + # just in case that dir was messed up (and we'll do a new + # copy). Not all of them, because it's pointless. + cands.sort() + cands.reverse() + + # mustn't be today: (strip those off) + while len(cands) > 0 and cands[0] == today: + cands = cands[1:] + + #print >>sys.stderr, 'cands=',cands + if len(cands) > 5: # truncate + cands = cands[0:5] + if len(cands) >= 1: + prior_dir_stuff = '--link-dest=' + prior_dir + '/' + string.join(cands, ' --link-dest='+prior_dir+'/') + #print >>sys.stderr, 'cands now=',prior_dir_stuff + RUN_HOST = ark.host.RUN_HOST() # the host we're on here... hosts_mgr = ark.host.ArkHostsMgr() *************** *** 152,169 **** if pretend or verbose: print >> sys.stderr, '[ -d %(dest_dir)s ] || %(MKDIR_P)s %(dest_dir)s' % vars() - if not pretend: - if not os.path.isdir(dest_dir): - if os.path.exists(dest_dir): - sys.stderr.write('Exists but isn\'t a directory:%s\n'%dest_dir) - sys.exit(1) - # mkdir -p then: - os.makedirs(dest_dir) - # and ensure permissions are what we want - # (ToDo: do this for every component in dest_dir that we mkdir??) - os.chown(dest_dir,replica_uid,replica_gid) - os.chmod(dest_dir,replica_mode) ! # ToDo: HACK HACK HACK (shouldn't have hard PYTHONPATH in there) ! if use_rdiff_backup: rsyncer_cmd = "%(RSYNCER)s %(rsyncer_verbose)s --remote-schema '%(SSH)s -x %%s /usr/bin/env PYTHONPATH=/our/share/ark:/our/.-ark-deploy/rdiff-backup--0.13.6/lib/python2.3/site-packages %(SRC_HOST_RSYNCER)s --server' %(excludes)s %(src_hostname)s::%(src_dir)s %(dest_dir)s"%vars() else: --- 188,210 ---- if pretend or verbose: print >> sys.stderr, '[ -d %(dest_dir)s ] || %(MKDIR_P)s %(dest_dir)s' % vars() ! if not pretend: ! dirs_to_make = [ dest_dir ] ! for d in dirs_to_make : ! if not os.path.isdir(d): ! if os.path.exists(d): ! sys.stderr.write('Exists but isn\'t a directory:%s\n'%d) ! sys.exit(1) ! # mkdir -p then: ! os.makedirs(d) ! # and ensure permissions are what we want ! # (ToDo: do this for every component in d that we mkdir??) ! os.chown(d,replica_uid,replica_gid) ! os.chmod(d,replica_mode) ! ! if use_dated_snapshots: ! rsyncer_cmd = '%(RSYNCER)s %(rsyncer_verbose)s %(rsyncer_pretend)s -e "%(SSH)s -x" --rsync-path=%(SRC_HOST_RSYNCER)s --bwlimit=100 --archive --delete %(excludes)s --delete-excluded %(prior_dir_stuff)s %(src_hostname)s:%(src_dir)s/ %(dest_dir)s'%vars() ! elif use_rdiff_backup: ! # ToDo: HACK HACK HACK (shouldn't have hard PYTHONPATH in there) rsyncer_cmd = "%(RSYNCER)s %(rsyncer_verbose)s --remote-schema '%(SSH)s -x %%s /usr/bin/env PYTHONPATH=/our/share/ark:/our/.-ark-deploy/rdiff-backup--0.13.6/lib/python2.3/site-packages %(SRC_HOST_RSYNCER)s --server' %(excludes)s %(src_hostname)s::%(src_dir)s %(dest_dir)s"%vars() else: |