[Assorted-commits] SF.net SVN: assorted: [209] auto-mirror/trunk/src/automirror.py

[<yan...@us...>]

Revision: 209
          http://assorted.svn.sourceforge.net/assorted/?rev=209&view=rev
Author:   yangzhang
Date:     2008-01-03 03:12:25 -0800 (Thu, 03 Jan 2008)

Log Message:
-----------
latest version of auto-mirror (abandoned)

Modified Paths:
--------------
    auto-mirror/trunk/src/automirror.py

Modified: auto-mirror/trunk/src/automirror.py
===================================================================
--- auto-mirror/trunk/src/automirror.py	2008-01-03 11:11:36 UTC (rev 208)
+++ auto-mirror/trunk/src/automirror.py	2008-01-03 11:12:25 UTC (rev 209)
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# vim:et:sw=2:ts=2
 
 """
 Mirrors a (one-level-deep) directory.
@@ -13,20 +14,36 @@
 
 import glob, ftplib, commands, socket, time, getopt, sys, os
 from signal import *
+from subprocess import *
+from traceback import *
 
+def log(*msg):
+  print ' '.join(map(str, (time.ctime(),) + msg))
+  sys.stdout.flush()
+
+def info(*msg): return log(*msg)
+def debug(*msg): return log(*msg)
+
 is_downloading = False
 
+def restart():
+  args = ['python'] + sys.argv
+  debug( 'spawning', args )
+  pid = os.spawnvp(os.P_NOWAIT, 'python', args)
+  sys.exit(0)
+  # debug( 'pid', pid )
+  # os.kill(os.getpid(), SIGTERM)
+
 def handle_sigusr1(sig, bt):
-  global is_downloading, debug
-  if debug: print 'got SIGUSR1'; sys.stdout.flush()
+  global is_downloading
+  debug( 'got SIGUSR1' )
   if is_downloading:
-    args = ['python'] + sys.argv
-    if debug: print 'spawning', args; sys.stdout.flush()
-    pid = os.spawnvp(os.P_NOWAIT, 'python', args)
-    if debug: print 'pid', pid; sys.stdout.flush()
-    os.kill(os.getpid(), SIGTERM)
+    restart()
 
+def handle_sigquit(sig, bt): print_stack( file = sys.stdout )
+
 signal(SIGUSR1, handle_sigusr1)
+signal(SIGQUIT, handle_sigquit)
 
 class DownloadFile( object ):
   def __init__(self, name):
@@ -62,7 +79,7 @@
 """
 
 def main():
-  global is_downloading, debug
+  global is_downloading
   updir = ".upload"
 
   server = user = passwd = nodedir = localpath = None
@@ -76,13 +93,13 @@
   # defaults
   run_forever = True
   timeout = 5
-  debug = False
+  do_debug = False
 
   # parse arguments
   for o, a in opts:
     if o == "-t": run_forever = False
     if o == "-r": timeout = int(a)
-    if o == "-v": debug = True
+    if o == "-v": do_debug = True
 
   ftp = None
 
@@ -94,10 +111,10 @@
     try:
       # get directory listing
       is_downloading = True
-      if debug: print "connecting.."; sys.stdout.flush()
+      debug( "connecting.." )
       ftp = ftplib.FTP(server, user, passwd)
       ftp.set_pasv(True)
-      if debug: print "cwd"; sys.stdout.flush()
+      debug( "cwd" )
       ftp.cwd(nodedir)
 
       # NOTE: ftplib has issues with nlst-ing empty dirs, so make sure on the
@@ -107,18 +124,18 @@
       rmflist = []
       acklist = []
       try:
-        if debug: print "get remflist"; sys.stdout.flush()
+        debug( "get remflist" )
         remflist = ftp.nlst("")
       except ftplib.all_errors, e:
         pass
 
       try:
-        if debug: print "get acklist"; sys.stdout.flush()
+        debug( "get acklist" )
         acklist = ftp.nlst(updir+"/")
       except ftplib.all_errors, e:
         pass
 
-      if debug: print "removing files..."; sys.stdout.flush()
+      debug( "removing files..." )
 
       # remove files that have been removed from the server
       locflist = os.listdir(localpath)
@@ -130,7 +147,7 @@
           except:
             pass # igorne missing ".md5" files for partial downloads
 
-      if debug: print 'fetching files...'; sys.stdout.flush()
+      debug( 'fetching files...' )
       for filename in remflist:
 
         # check if file had already been downloaded
@@ -141,7 +158,7 @@
           localpname = localpath+"/"+filename
           df = DownloadFile(localpname)
 
-          if debug: print "before download of "+filename; sys.stdout.flush()
+          debug( "before download of "+filename )
 
           # download file
           ftp.retrbinary("RETR %s" % filename,
@@ -149,7 +166,7 @@
                          16*1024,
                          df.get_offset())
 
-          if debug: print "after download of "+filename; sys.stdout.flush()
+          debug( "after download of "+filename )
 
           # fix the timestamp, which is often way in the future (for some
           # reason, the cabs don't quite have the right date/time)
@@ -167,9 +184,9 @@
           # remove md5 file
           commands.getoutput("rm %s" % md5fname)
 
-          print "[%s] download complete" % filename
+          info( "[%s] download complete" % filename )
 
-      if debug: print 'done downloading'; sys.stdout.flush()
+      debug( 'done downloading' )
 
       # done with all files
       is_downloading = False
@@ -183,29 +200,44 @@
       ftp.storbinary("STOR .upload/ls.txt", lsf)
       lsf.close()
 
+      # launch a new download.py if avail
+      downloads = sorted(glob.glob(localpath + '/download.py-*'))
+      if len(downloads)>0:
+        shutil.move(downloads[-1], '/opt/bin/download.py')
+        restart()
+
       # if there are Makefiles, execute lexicographically last one
       makefiles = sorted(glob.glob(localpath + '/Makefile*'))
       if len(makefiles)>0:
+        # note that os.system defers interrupts, for some reason
         os.system('make -C ' + localpath + ' -f ' + makefiles[-1])
+        #p = Popen('make -C ' + localpath + ' -f ' + makefiles[-1],
+        #          shell = True)
+        #debug( 'waiting on pid', p.pid )
+        #try: os.waitpid( p.pid, 0 )
+        #except: pass # TODO retry
 
+      debug( 'sleeping' )
+
       if run_forever: time.sleep(30)
       else: done = True
 
+      debug( 'looping' )
+
     except ftplib.all_errors, e:
-      print "FTP error", e
+      info( "FTP error", e )
       is_downloading = False
       time.sleep(timeout)
-      print "Retrying.."
+      info( "Retrying.." )
 
     except socket.error, e:
-      print "Socket error", e
+      info( "Socket error", e )
       is_downloading = False
       time.sleep(timeout)
-      print "Retrying.."
+      info( "Retrying.." )
 
   ftp.quit()
 
 if __name__ == "__main__":
   main()
 
-# vim:et:sw=2:ts=2


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.