CVS: sidai/dchunk-utils acct-for-diskspace,1.16,1.17 sync-replica,1.10,1.11

SourceForge Headquarters 1320 Columbia Street Suite 310 San Diego, CA 92101 +1 (858) 422-6466

Update of /cvsroot/ark/sidai/dchunk-utils
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv28042/dchunk-utils

Modified Files:
	acct-for-diskspace sync-replica 
Log Message:
updates

Index: acct-for-diskspace
===================================================================
RCS file: /cvsroot/ark/sidai/dchunk-utils/acct-for-diskspace,v
retrieving revision 1.16
retrieving revision 1.17
diff -C2 -r1.16 -r1.17
*** acct-for-diskspace	4 Apr 2005 10:09:38 -0000	1.16
--- acct-for-diskspace	7 Feb 2006 11:40:21 -0000	1.17
***************
*** 71,75 ****
  """

! import os,re,string,sys
  import ark.control
  import ark.dchunk
--- 71,75 ----
  """

! import errno,os,re,string,sys
  import ark.control
  import ark.dchunk
***************
*** 208,213 ****
  	    (device,mountpt,mttype) = match.group(1,2,3)

! 	    # print l,
!             # print '<',device,'><',mountpt, '><',mttype, '>' 

  	    # we're only interested in real, live disk mounts,
--- 208,213 ----
  	    (device,mountpt,mttype) = match.group(1,2,3)

! 	    #print l,
!             #print '<',device,'><',mountpt, '><',mttype, '>' 

  	    # we're only interested in real, live disk mounts,
***************
*** 261,265 ****
  def drillDown(level,host,dir):
      hostname = host._name
!     #sys.stderr.write('lev%d: %s:%s\n'%(level,hostname,dir))
      if ACCTD.has_key(hostname) and ACCTD[hostname].has_key(dir):
  	ACCTD[hostname][dir] = 2 # done
--- 261,265 ----
  def drillDown(level,host,dir):
      hostname = host._name
!     # sys.stderr.write('lev%d: %s:%s\n'%(level,hostname,dir))
      if ACCTD.has_key(hostname) and ACCTD[hostname].has_key(dir):
  	ACCTD[hostname][dir] = 2 # done
***************
*** 273,276 ****
--- 273,286 ----
              print 'f %s:%s' % (hostname,dir)
              return
+         except OSError,e:
+             if e.errno == errno.EACCES:
+                 print 'P %s:%s # permissions...' % (hostname,dir)
+                 return
+             else:
+ #               print e.errno
+                 raise # again
+         except:
+ #            print `e`
+             raise

          if not in_play:
***************
*** 283,287 ****

  	for sd in substuff:
! 	    dsd = '%s/%s' % (dir,sd)
  	    if IGNORE_DIRS.has_key(sd): continue # specifically uninteresting
              if not in_play:
--- 293,300 ----

  	for sd in substuff:
!             if dir == '/':
!                 dsd = '/%s' % (sd)
!             else:
!                 dsd = '%s/%s' % (dir,sd)
  	    if IGNORE_DIRS.has_key(sd): continue # specifically uninteresting
              if not in_play:

Index: sync-replica
===================================================================
RCS file: /cvsroot/ark/sidai/dchunk-utils/sync-replica,v
retrieving revision 1.10
retrieving revision 1.11
diff -C2 -r1.10 -r1.11
*** sync-replica	30 Jul 2005 12:35:33 -0000	1.10
--- sync-replica	7 Feb 2006 11:40:21 -0000	1.11
***************
*** 2,6 ****
  ################################################################
  # 
! # Copyright (c) 2002-2004, Will Partain
  # All rights reserved.
  # 
--- 2,6 ----
  ################################################################
  # 
! # Copyright (c) 2002-2005, Will Partain
  # All rights reserved.
  # 
***************
*** 55,67 ****
      --help       Print this message and exit.
      --rdiff-backup Use rdiff-backup instead of rsync.
      --rsyncer-verbose Run rsync/rdiff-backup verbosely.

  """

! import os,re,sys
  #import os,re,string,sys
  #import ark.control
  import ark.dchunk
  import ark.host
  #import ark.error

--- 55,70 ----
      --help       Print this message and exit.
      --rdiff-backup Use rdiff-backup instead of rsync.
+     --dated-snapshots Do dated snapshots with rsync (instead
+                  of plain rsync or rdiff-backup)
      --rsyncer-verbose Run rsync/rdiff-backup verbosely.

  """

! import datetime,os,re,string,sys
  #import os,re,string,sys
  #import ark.control
  import ark.dchunk
  import ark.host
+ import ark.utils # globNormal
  #import ark.error

***************
*** 77,80 ****
--- 80,84 ----
  	, 'pretend'
  	, 'rdiff-backup'
+ 	, 'dated-snapshots'
  	, 'rsyncer-verbose'
          ]
***************
*** 84,88 ****
      if opts.isSet('quiet'): verbose = None
      pretend = opts.isSet('pretend')
!     use_rdiff_backup = opts.isSet('rdiff-backup')

      non_opts = opts.cmdLineNonOptions()
--- 88,97 ----
      if opts.isSet('quiet'): verbose = None
      pretend = opts.isSet('pretend')
!     use_rdiff_backup    = opts.isSet('rdiff-backup')
!     use_dated_snapshots = opts.isSet('dated-snapshots')
! 
!     if use_rdiff_backup and use_dated_snapshots:
!         sys.stderr.write('Only one of: --rdiff-backup, --dated-snapshots\n')
!         sys.exit(1)

      non_opts = opts.cmdLineNonOptions()
***************
*** 117,120 ****
--- 126,156 ----
      (dest_hostname,dest_dir,nfs4_junk_inFo) = replica.masterHostnameAndDir()

+     if use_dated_snapshots:
+         prior_dir = dest_dir # remember this root...
+         prior_dir_stuff = ''
+         today     = datetime.date.today().isoformat() # YYYY-MM-DD
+         dest_dir += '/%s' % (today)
+ 
+         cands = ark.utils.globNormal(prior_dir,host_spec='.',globb='????-??-??')
+         #print >>sys.stderr, 'cands=',cands
+ 
+         # there could be zero to many candidates; we want --link-dest args
+ 	# for (up to) the last five [arbitrary] candidates.  Not one,
+ 	# just in case that dir was messed up (and we'll do a new
+ 	# copy).  Not all of them, because it's pointless.
+         cands.sort()
+         cands.reverse()
+ 
+         # mustn't be today: (strip those off)
+         while len(cands) > 0 and cands[0] == today:
+             cands = cands[1:]
+ 
+         #print >>sys.stderr, 'cands=',cands
+         if len(cands) > 5: # truncate
+             cands = cands[0:5]
+ 	if len(cands) >= 1:
+             prior_dir_stuff = '--link-dest=' + prior_dir + '/' + string.join(cands, ' --link-dest='+prior_dir+'/')
+         #print >>sys.stderr, 'cands now=',prior_dir_stuff
+ 
      RUN_HOST  = ark.host.RUN_HOST() # the host we're on here...
      hosts_mgr = ark.host.ArkHostsMgr()
***************
*** 152,169 ****
      if pretend or verbose:
          print >> sys.stderr, '[ -d %(dest_dir)s ] || %(MKDIR_P)s %(dest_dir)s' % vars()
-     if not pretend:
-         if not os.path.isdir(dest_dir):
-             if os.path.exists(dest_dir):
-                 sys.stderr.write('Exists but isn\'t a directory:%s\n'%dest_dir)
-                 sys.exit(1)
-             # mkdir -p then:
-             os.makedirs(dest_dir)
-             # and ensure permissions are what we want
-             # (ToDo: do this for every component in dest_dir that we mkdir??)
-             os.chown(dest_dir,replica_uid,replica_gid)
-             os.chmod(dest_dir,replica_mode)

! # ToDo: HACK HACK HACK (shouldn't have hard PYTHONPATH in there)
!     if use_rdiff_backup:
          rsyncer_cmd = "%(RSYNCER)s %(rsyncer_verbose)s --remote-schema '%(SSH)s -x %%s /usr/bin/env PYTHONPATH=/our/share/ark:/our/.-ark-deploy/rdiff-backup--0.13.6/lib/python2.3/site-packages %(SRC_HOST_RSYNCER)s --server' %(excludes)s %(src_hostname)s::%(src_dir)s %(dest_dir)s"%vars()
      else:
--- 188,210 ----
      if pretend or verbose:
          print >> sys.stderr, '[ -d %(dest_dir)s ] || %(MKDIR_P)s %(dest_dir)s' % vars()

!     if not pretend:
!         dirs_to_make = [ dest_dir ]
!         for d in dirs_to_make :
!             if not os.path.isdir(d):
!                 if os.path.exists(d):
!                     sys.stderr.write('Exists but isn\'t a directory:%s\n'%d)
!                     sys.exit(1)
!                 # mkdir -p then:
!                 os.makedirs(d)
!                 # and ensure permissions are what we want
!                 # (ToDo: do this for every component in d that we mkdir??)
!                 os.chown(d,replica_uid,replica_gid)
!                 os.chmod(d,replica_mode)
! 
!     if use_dated_snapshots:
!         rsyncer_cmd = '%(RSYNCER)s %(rsyncer_verbose)s %(rsyncer_pretend)s -e "%(SSH)s -x" --rsync-path=%(SRC_HOST_RSYNCER)s --bwlimit=100 --archive --delete %(excludes)s --delete-excluded %(prior_dir_stuff)s %(src_hostname)s:%(src_dir)s/ %(dest_dir)s'%vars()
!     elif use_rdiff_backup:
!         # ToDo: HACK HACK HACK (shouldn't have hard PYTHONPATH in there)
          rsyncer_cmd = "%(RSYNCER)s %(rsyncer_verbose)s --remote-schema '%(SSH)s -x %%s /usr/bin/env PYTHONPATH=/our/share/ark:/our/.-ark-deploy/rdiff-backup--0.13.6/lib/python2.3/site-packages %(SRC_HOST_RSYNCER)s --server' %(excludes)s %(src_hostname)s::%(src_dir)s %(dest_dir)s"%vars()
      else: