SF.net SVN: safekeep:[622] safekeep/trunk

[<di...@us...>]

Revision: 622
          http://safekeep.svn.sourceforge.net/safekeep/?rev=622&view=rev
Author:   dimi
Date:     2008-11-19 18:16:58 +0000 (Wed, 19 Nov 2008)

Log Message:
-----------
First cut at implementing bandwidth limiting based on trickle.

Modified Paths:
--------------
    safekeep/trunk/TODO
    safekeep/trunk/safekeep
    safekeep/trunk/safekeep.spec.in

Modified: safekeep/trunk/TODO
===================================================================
--- safekeep/trunk/TODO	2008-11-19 16:39:56 UTC (rev 621)
+++ safekeep/trunk/TODO	2008-11-19 18:16:58 UTC (rev 622)
@@ -5,7 +5,6 @@
   * Avoid snapshotting snapshots
   * Don't snapshot a device if a snapshot is already present
   * Protect against multiple safekeep instance running at once
-  * Use -l with ssh to limit the bandwidth used during the backup
 
 Feedback from users:
   * Øyvind Skaar <os...@op...>: FreeBSD have snapshot capabilities

Modified: safekeep/trunk/safekeep
===================================================================
--- safekeep/trunk/safekeep	2008-11-19 16:39:56 UTC (rev 621)
+++ safekeep/trunk/safekeep	2008-11-19 18:16:58 UTC (rev 622)
@@ -45,14 +45,17 @@
 
 config_file = '/etc/safekeep/safekeep.conf'
 config_ext = '.backup'
+trickle_cmd = 'trickle'
 logbuf = []
 is_client = False
 verbosity_level = 1
 verbosity_ssh = ''
+verbosity_trickle = ''
 work_user = getpass.getuser()
 backup_user = None
 home_dir = None
 base_dir = None
+default_bandwidth = {}
 
 PROTOCOL = "1.1"
 VERSION = "1.0.5"
@@ -99,7 +102,7 @@
                 if line.startswith(marker):
                     marker = None
                 continue
-            if (line.startswith("Errors ")):
+            if line.startswith("Errors "):
                 errs = int(line[6:])
             info(line.rstrip())
     finally:
@@ -118,6 +121,17 @@
 def error(msg):
     log(msg, 'ERR')
 
+def try_to_run(cmd):
+    cmd = cmd.split(' ')[0]
+    proc = popen2.Popen4(args)
+    proc.tochild.close()
+    for line in proc.fromchild:
+        info(line.rstrip())
+    proc.fromchild.close()
+    rc = proc.wait()
+
+    return os.WIFEXITED(rc)
+
 def spawn(args):
     if isinstance(args, str) or isinstance(args, unicode):
         debug('Run [' + args + ']')
@@ -239,6 +253,13 @@
         raise ConfigException('Empty ' + clude_el.tagName)
     return { 'type' : clude_el.tagName, 'path' : path, 'glob' : glob, 'regexp' : regexp }
 
+def parse_bandwidth(bw_el):
+    return {
+        'overall': int(bw_el.getAttribute('overall') or 0),
+        'download': int(bw_el.getAttribute('download') or 0),
+        'upload': int(bw_el.getAttribute('upload') or 0)
+    }
+
 def parse_config(backup_el, dflt_id):
     if backup_el.tagName != 'backup':
         raise ConfigException('Invalid config file, the top level element must be <backup>')
@@ -264,6 +285,13 @@
     if key_data and not os.path.isabs(key_data):
         key_data = os.path.join(home_dir, key_data)
 
+    bw = {}
+    bw_el = backup_el.getElementsByTagName('bandwidth')
+    if len(bw_el) == 1:
+        bw = parse_bandwidth(bw_el[0])
+    elif len(bw_el) > 1:
+        raise ConfigException('Can not have more than a bandwidth element')
+
     repo_el = backup_el.getElementsByTagName('repo')
     dir = None
     retention = None
@@ -323,7 +351,7 @@
 
     return { 'id': id, 'host' : host, 'user' : user, 'key_ctrl' : key_ctrl, 'key_data' : key_data,
              'dir' : dir, 'retention' : retention, 'dumps' : dumps, 'snaps' : snaps,
-             'cludes' : cludes, 'options' : options}
+             'cludes' : cludes, 'options' : options, 'bw': bw}
 
 def parse_locs(cfglocs):
     cfgfiles = []
@@ -709,13 +737,35 @@
 def do_server_rdiff(cfg, bdir, nice, force):
     args = []
 
-    if (nice)
+    if nice:
         args.extend(['nice', '-n' + nice])
 
     args.extend(['rdiff-backup'])
 
     if cfg['host']:
-        schema = 'ssh %s -i %s %%s rdiff-backup --server' % (verbosity_ssh, cfg['key_data'])
+        trickle = ''
+
+        def get_bw(vals, dir):
+            return vals.get(dir) or vals.get('overall')
+
+        def get_bandwidth(cfg, dir):
+            return get_bw(cfg['bw'], dir) or get_bw(default_bandwidth, dir)
+
+        limit_dl = get_bandwidth(cfg, 'download')
+        limit_ul = get_bandwidth(cfg, 'upload')
+        if limit_dl or limit_ul:
+            trickle = trickle_cmd + ' ' + verbosity_trickle
+            if limit_dl:
+                trickle += ' -d ' + limit_dl
+            if limit_ul:
+                trickle += ' -u ' + limit_ul
+
+        if trickle:
+            if !try_to_run(trickle_cmd + ' -V'):
+                warn('Trickle not available, bandwidth limiting disabled')
+                trickle = ''
+
+        schema = '% ssh %s -i %s %%s rdiff-backup --server' % (trickle, verbosity_ssh, cfg['key_data'])
         args.extend(['--remote-schema', schema])
 
     if force:
@@ -1227,6 +1277,13 @@
             else:
                  cfgfile = config_file
             props = {}
+
+        def get_int(prop):
+            v = props.get(p)
+            if v is not None and v is not '':
+                return int(v)
+            return None
+
         if 'backup.user' in props:
             backup_user = props['backup.user']
         if 'base.dir' in props:
@@ -1235,10 +1292,13 @@
             smtp = props['email.smtp.server']
         if 'email.to' in props:
             email = props['email.to'].split(',')
-        if 'nice.adjustment' in props:
-            nice_srv = props['nice.adjustment']
-            if (nice_srv) nice_srv = int(nice_srv)
+        nice_srv = get_int('nice.adjustment')
 
+        global default_bandwidth
+        default_bandwidth['overall'] = get_int('bandwidth.limit') or 0
+        default_bandwidth['download'] = get_int('bandwidth.limit.download') or 0
+        default_bandwidth['upload'] = get_int('bandwidth.limit.upload') or 0
+
         if len(cfglocs) == 0: 
             locs = os.path.join(os.path.dirname(cfgfile), 'backup.d')
             if os.path.isdir(locs): cfglocs.append(locs)
@@ -1274,10 +1334,10 @@
         if not ok: sys.exit(2)
 
     try:
-        global is_client, verbosity_level, verbosity_ssh
+        global is_client, verbosity_level, verbosity_ssh, verbosity_trickle
 
         if verbosity > 0:
-            verbosity_ssh = '-' + verbosity * 'v'
+            verbosity_trickle = verbosity_ssh = '-' + verbosity * 'v'
         if mode is 'server':
             is_client = False
             verbosity_level = 1 + verbosity

Modified: safekeep/trunk/safekeep.spec.in
===================================================================
--- safekeep/trunk/safekeep.spec.in	2008-11-19 16:39:56 UTC (rev 621)
+++ safekeep/trunk/safekeep.spec.in	2008-11-19 18:16:58 UTC (rev 622)
@@ -116,6 +116,7 @@
 %doc sample.backup
 
 %changelog
+  - Implement bandwidth limiting, based on trickle
   - By default, run safekeep with nice +10 on the server side
   - Run ssh/rdiff through nice so we can control the load better on the server
   - Do not compress the SSH traffic, it is handled by rdiff-backup


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.