Commit [r43] Maximize Restore History

introduced server TMMS which handels all running jobs an a specific host

hotbdesiato 2012-10-01

added /trunk/bin/TMMS
changed /trunk/README
changed /trunk/etc/TaskDispatcher.info
changed /trunk/UserServer/TMS.py
changed /trunk/bin/hRunJob
changed /trunk/Server/TaskDispatcher.py
changed /trunk/scripts/mkDist.py
changed /trunk/bin/TMS
changed /trunk/Client/hRunJob.py
copied /trunk/UserServer/TMM.py -> /trunk/UserServer/TMMS.py
/trunk/bin/TMMS Diff Switch to side-by-side view
Loading...
/trunk/README Diff Switch to side-by-side view
Loading...
/trunk/etc/TaskDispatcher.info Diff Switch to side-by-side view
Loading...
/trunk/UserServer/TMS.py Diff Switch to side-by-side view
Loading...
/trunk/bin/hRunJob Diff Switch to side-by-side view
Loading...
/trunk/Server/TaskDispatcher.py Diff Switch to side-by-side view
Loading...
/trunk/scripts/mkDist.py Diff Switch to side-by-side view
Loading...
/trunk/bin/TMS Diff Switch to side-by-side view
Loading...
/trunk/Client/hRunJob.py Diff Switch to side-by-side view
Loading...
/trunk/UserServer/TMM.py to /trunk/UserServer/TMMS.py
--- a/trunk/UserServer/TMM.py
+++ b/trunk/UserServer/TMMS.py
@@ -1,4 +1,4 @@
-# user specific task manager menial (TMM) server which is dynamically started by TMS on each host of cluster
+# user specific taskmanager menial server (TMMS) which is dynamically started by TMS on each host of cluster
 
 import socket
 import SocketServer
@@ -24,6 +24,7 @@
 import xml.dom.minidom
 import exceptions
 import json
+import tempfile
 
 homedir = os.environ['HOME']
 user = pwd.getpwuid(os.getuid())[0]
@@ -55,7 +56,7 @@
 binPath    = '%s/bin' % tmpath	# for hRun
 serverPath = '%s/UserServer' % tmpath
 etcPath    = '%s/etc'    % tmpath	# for TaskDispatcher.info
-libPath  = '%s/lib' % tmpath		# for hSocket
+#libPath  = '%s/lib' % tmpath		# for hSocket
 
 # ssl configuration
 certfile = "%s/.taskManager/%s.crt" % (homedir,user)
@@ -63,87 +64,42 @@
 ca_certs = "%s/.taskManager/ca_certs.%s.crt" % (homedir,user)
 
 
-sys.path.insert(0,libPath)
+#sys.path.insert(0,libPath)
 
 
 from hSocket import hSocket
 from hTMConnections import TMConnection
 
-# read config file for
-#   TaskDispatcherHost
-#   TaskDispatcherPort
-f=file('%s/TaskDispatcher.info' % etcPath)
-c=compile(f.read(),'config','exec')
-f.close()
-exec c
-
-# set as default host
-TaskDispatcherHostDefault = copy(TaskDispatcherHost)
-TaskDispatcherPortDefault = copy(TaskDispatcherPort)
-
-
-progName = "TMS.py"
+progName = "TMMS.py"
 def printHelp(where=sys.stdout):
-    where.write("NAME              %s - task manager server\n" % progName)
+    where.write("NAME              %s - taskmanager menial server\n" % progName)
     where.write("\n")
     where.write("SYNOPSIS          %s -h\n" % progName)
-    where.write("                  %s [OPTION] Port\n" % progName)
+    where.write("                  %s [OPTION] Port TMSHost TMSPort\n" % progName)
     where.write("\n")
-    where.write("DESCRIPTION       Starts a taskmanager server on PORT.\n")
+    where.write("DESCRIPTION       Starts a taskmanager menial server on PORT. Communicate with\n")
+    where.write("                  TaskManagerServer on TMSHost:TMSPort.\n")
     where.write("\n")
     where.write("OPTIONS\n")
-    where.write("   -d             Run TMS not as daemon.\n")
+    where.write("   -d             Run TMMS in non-daemon mode.\n")
     where.write("   -h             Print this help.\n")
     where.write("   -l LOGFILE     Write in and out communications in LOGFILE.\n")
-    where.write("   -t             Do not terminate TMS after jobs has been processed.\n")
     where.write("   -v             Verbose mode. Print status information on stdout.\n")
     where.write("\n")
-    where.write("AUTHOR            Written by Hendrik Hache (hache@molgen.mpg.de)\n")
-    where.write("                             Svetlana Mareva (mareva@molgen.mpg.de)\n")
-
-
-class Job:
-    def __init__(self):
-        self.status = 0
-        self.jobInfo = {
-            'command': None,
-            'host': None,
-            'shell': None,
-            'TaskDispatcherHost': None,
-            'TaskDispatcherPort': None,
-            'pid': None,
-            'returnCode': None
-            }
-
-    def getStatus(self):
-        return self.status
-
-    def getJobInfo(self,what):
-        return self.jobInfo.get(what,None)
-
-    def setAsAdded(self):
-        self.status += 1
-
-    def setAsStartRequested(self):
-        self.status += 2
-
-    def setAsStarted(self):
-        self.status += 4
-
-    def setAsFinished(self):
-        self.status += 8
-
-    def setAsStartFailed(self):
-        self.status = 1
-
-    def setJobInfo(self,**kwargs):
-        for k in kwargs:
-            if k in self.jobInfo:
-                self.jobInfo[k]=kwargs[k]
-
-
-# TaskManagerServer is a TCPServer using the threading mix in to create a new thread for every request.
-class TaskManagerServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer, Daemon):
+    where.write("AUTHOR            Written by hotbdesiato.\n")
+
+class Job():
+    """ structur for collecting job specific info """
+    def __init__(self,jobID,command,jobInfo=None,logFile=None,shell=None,pid=None):
+        self.jobID = jobID
+        self.command = command
+        self.jobInfo = jobInfo
+        self.logFile = logFile
+        self.shell = shell
+        self.pid = pid
+
+# TaskManagerMenail Server is a TCPServer using the threading mix in to create a new thread for every request.
+class TaskManagerMenialServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer, Daemon):
     # This means the main server will not do the equivalent of a
     # pthread_join() on the new threads.  With this set, Ctrl-C will
     # kill the server reliably.
@@ -156,11 +112,11 @@
     allow_reuse_address = True
     reuest_queue_size = 10
     
-    #def __init__(self,host,port,user,verboseMode=False,doNotTerminate=False,logFileTMS=None):
     def __init__(self,
-                 user,
                  host,
                  port,
+                 tmsHost,
+                 tmsPort,
                  handler,
                  processor,
                  sslConnection=False,
@@ -168,14 +124,15 @@
                  certfile=None,
                  ca_certs=None,
                  verboseMode=False,
-                 doNotTerminate=False,
-                 logFileTMS=None):
+                 logFileTMMS=None):
         Daemon.__init__(self,'/tmp/daemon-example.pid')
         self.startTime = strftime("%d %b %Y, %H.%M.%S") # strftime("%Y %m %d %H.%M.%S")
         self.ID = int(time())
         self.info = {}
         self.host=host
         self.port=port
+        self.tmsHost=tmsHost
+        self.tmsPort=tmsPort
         self.user=user
         self.setHost(host)
         self.setPort(port)
@@ -194,40 +151,18 @@
         self.logFiles = {}	# job specific logfiles: {jobID: logfile, ...}  dict of all logfiles
         self.runningJobs = []
 
-        self.terminateLater=False 	# prevent termination after certain commands
-        self.terminate=False		# terminate TMS
-        self.doNotTerminate=doNotTerminate	# do not terminate TMS
-        
         self.verboseMode = verboseMode
-        self.logFileTMS = logFileTMS
+        self.logFileTMMS = logFileTMMS
 
         # start the server
         SocketServer.TCPServer.__init__(self, (self.host,self.port), handler)
 
         if self.verboseMode:
             sys.stdout.write("\n")
-            sys.stdout.write('TaskManagerServer has been started on %s:%s\n' % (self.host,self.port))
+            sys.stdout.write('TaskManagerMenialServer has been started on %s:%s\n' % (self.host,self.port))
             sys.stdout.write("\n")
             sys.stdout.flush()
 
-    def checkJobID(self,jobID):
-        # check if job with jobID is already known
-        # jobID: "USER.TD_ID.TMS_ID.JOB_ID"
-        user,tdID,tmsID,jID = jobID.split('.')
-
-        tmsID = int(tmsID)
-        if tmsID != self.ID:
-            # job comes from a different TMS
-            status = 0
-        elif jobID not in self.jobsDict:
-            # job comes from this TMS but is unknown
-            status = 1
-        else:
-            # job is known by this TMS
-            status = 2
-
-        return status
-    
     def setHost(self,host):
         self.info['host']=host
 
@@ -245,9 +180,11 @@
 
         try:
             self.serve_forever()
+            
             # wait for all active threads (except main thread) until it terminates
             for currThread in threading.enumerate()[1:threading.activeCount()]:
                 currThread.join()
+                
             print "... done"
                     
         except KeyboardInterrupt:
@@ -255,32 +192,24 @@
             
 
 # The RequestHandler handles an incoming request.
-class TaskManagerServerHandler(SocketServer.BaseRequestHandler):
-    def __init__(self, request, clientAddress, TMS):
+class TaskManagerMenialServerHandler(SocketServer.BaseRequestHandler):
+    def __init__(self, request, clientAddress, TMMS):
         self.request = request
         self.requestHost,self.requestPort = self.request.getpeername()
-        self.TMS = TMS
-        self.terminateLater = False
+        self.TMMS = TMMS
         self.currThread = threading.currentThread()
 
-        SocketServer.BaseRequestHandler.__init__(self, request, clientAddress, self.TMS)
-
-    def finish(self):
-        # check for termination of TMS
-        if not self.TMS.doNotTerminate and (not self.terminateLater and self.TMS.runningJobs==[] and self.TMS.jobsDict=={} and self.TMS.infoSockets==[] and self.TMS.logFiles=={} and threading.activeCount()==2):
-            if self.TMS.verboseMode:
-                print "last thread (%s) has been deleted" % threading.currentThread().getName()
-                print "server shutdown ..."
-            self.TMS.shutdown()
+        SocketServer.BaseRequestHandler.__init__(self, request, clientAddress, self.TMMS)
+
+    #def finish(self):
+    #    pass
 
     def handle(self):
-        persistentSocket = False
-        firstRequest = True
-
-        if self.TMS.verboseMode:
-            sys.stdout.write("[%s] TMS (%s:%s) has created a new thread (%s) for %s:%s \n" % (datetime.now().strftime("%Y.%m.%d %H:%M:%S"),
-                                                                                              self.TMS.host,
-                                                                                              self.TMS.port,
+        if self.TMMS.verboseMode:
+            sys.stdout.write("-----------------------\n")
+            sys.stdout.write("[%s] TMMS (%s:%s) has created a new thread (%s) for %s:%s \n" % (datetime.now().strftime("%Y.%m.%d %H:%M:%S"),
+                                                                                              self.TMMS.host,
+                                                                                              self.TMMS.port,
                                                                                               self.currThread.getName(),
                                                                                               self.requestHost,
                                                                                               self.requestPort))
@@ -288,54 +217,50 @@
             sys.stdout.write("[%s] number of current threads: %s\n" % (datetime.now().strftime("%Y.%m.%d %H:%M:%S"),threading.activeCount()))
             sys.stdout.flush()
 
-        while firstRequest or persistentSocket:
-            firstRequest = False
-            (sread, swrite, sexc) = select.select([self.request], [], [], None)
-            
-            if self.TMS.verboseMode:
-                sys.stdout.write("-----------------------\n")
-                sys.stdout.write("[%s] [%s:%s] Thread (%s) got new request\n" % (datetime.now().strftime("%Y.%m.%d %H:%M:%S"),self.requestHost,self.requestPort,self.currThread.getName()))
-                sys.stdout.flush()
-
-            if not persistentSocket:
-                # instantiate new socket
-                hSock = hSocket(sock=self.request,
-                                serverSideSSLConn=True,
-                                sslConnection=TMS.sslConnection,
-                                certfile=TMS.certfile,
-                                keyfile=TMS.keyfile,
-                                ca_certs=TMS.ca_certs)
-                
-            receivedStr = hSock.recv()
-
-            # process request
-            try:
-                persistentSocket, self.terminateLater = self.TMS.processor.process(receivedStr, hSock, persistentSocket, self.TMS)
-            except:
-                # processing failed
-                tb = sys.exc_info()
-                    
-                sys.stdout.write('[%s] [%s:%s] Error while processing request!\n' % (datetime.now().strftime("%Y.%m.%d %H:%M:%S"),self.requestHost,self.requestPort))
-                sys.stdout.flush()
-
-                # maybe output to stderr?
-                traceback.print_exception(*tb,file=sys.stdout)
-                sys.stdout.flush()
-
-                hSock.send("Error while processing request!\n%s" %  tb[1])
-
-            if self.TMS.verboseMode:
-                sys.stdout.write("-----------------------\n")
-                sys.stdout.flush()
-
-        if self.TMS.verboseMode:
-            sys.stdout.write("[%s] TMS (%s:%s) has deleted thread (%s)\n" % (datetime.now().strftime("%Y.%m.%d %H:%M:%S"),
-                                                                             self.TMS.host,
-                                                                             self.TMS.port,
-                                                                             self.currThread.getName()))
+        # waits for event on request socket
+        (sread, swrite, sexc) = select.select([self.request], [], [], None)
+        
+        if self.TMMS.verboseMode:
+            sys.stdout.write("-----------------------\n")
+            sys.stdout.write("[%s] [%s] got new request\n" % (datetime.now().strftime("%Y.%m.%d %H:%M:%S"),
+                                                              self.currThread.getName()))
+            sys.stdout.flush()
+
+        hSock = hSocket(sock=self.request,
+                        serverSideSSLConn=True,
+                        sslConnection=TMMS.sslConnection,
+                        certfile=TMMS.certfile,
+                        keyfile=TMMS.keyfile,
+                        ca_certs=TMMS.ca_certs)
+
+        receivedStr = hSock.recv()
+
+        try:
+            self.TMMS.processor.process(receivedStr, hSock, self.TMMS)
+        except:
+            # processing failed
+            tb = sys.exc_info()
+
+            sys.stdout.write('[%s] [%s:%s] Error while processing request!\n' % (datetime.now().strftime("%Y.%m.%d %H:%M:%S"),self.requestHost,self.requestPort))
+            sys.stdout.flush()
+
+            traceback.print_exception(*tb,file=sys.stderr)
+            sys.stdout.flush()
+
+            hSock.send("Error while processing request!\n%s" %  tb[1])
+
+        if self.TMMS.verboseMode:
+            sys.stdout.write("-----------------------\n")
+            sys.stdout.write("[%s] TMMS (%s:%s) has deleted %s\n" % (datetime.now().strftime("%Y.%m.%d %H:%M:%S"),
+                                                                    self.TMMS.host,
+                                                                    self.TMMS.port,
+                                                                    self.currThread.getName()))
+            sys.stdout.write("[%s] number of current threads: %s\n" % (datetime.now().strftime("%Y.%m.%d %H:%M:%S"),
+                                                                       threading.activeCount()-1))
             sys.stdout.write("-----------------------\n")
             sys.stdout.flush()
-            
+
+
         # closing socket
         try:
             hSock.close()
@@ -346,55 +271,29 @@
 
 
 
-class TaskManagerServerProcessor:
-    #def __init__(self,desObj):
-    #    self.desObj = desObj
-    
-    def process(self, s, request, persistentSocket, TMS):
+class TaskManagerMenialServerProcessor:
+    def process(self, s, request, TMMS):
         """process request string"""
         h,p = request.socket.getpeername()
         currThread = threading.currentThread()
-        
-        #self.broadcastString('[%s:%s] number active threads %s' % (h,p,threading.activeCount()),TMS)
-        #self.broadcastString('[%s:%s] current thread %s' % (h,p,currThread.getName()),TMS)
-        self.broadcastString('[%s:%s] receiving ...' % (h,p),TMS)
+        threadName = currThread.getName()
+        
+        self.broadcastString('[%s] receiving ...' % threadName,TMMS)
 
         terminateLater = False
         
-        self.broadcastString('[%s:%s] ... IN: %s' % (h,p,s),TMS)
-        self.logOutput('[%s:%s] IN: %s' % (h,p,s),TMS)
+        self.broadcastString('[%s] ... %s' % (threadName,s),TMMS)
+        self.logOutput('[%s] IN: %s' % (threadName,s),TMMS)
             
         receivedStr = s
 
         if not receivedStr:
-            self.broadcastString('[%s:%s] ... socket has been closed' % (h,p),TMS)
+            self.broadcastString('[%s] ... socket has been closed' % threadName,TMMS)
             return False, False
-
-        #####################
-        # grep TDHost and TDPort
-        reTD = re.compile('^TD:([^:]*):([^:]*)(@.*)$')
-        if reTD.match(receivedStr):
-            TDHost = reTD.match(receivedStr).groups()[0]
-            TDPort = int(reTD.match(receivedStr).groups()[1])
-            receivedStr = reTD.match(receivedStr).groups()[2]
-        else:
-            # use global definition
-            TDHost = copy(TaskDispatcherHostDefault)
-            TDPort = copy(TaskDispatcherPortDefault)
-
 
         #####################
         # check if request is a known command
         
-        ########
-        # @check
-        #    response an ok
-        #    do not terminate task manager server
-        if re.match('^@check$',receivedStr):
-            request.send("ok")
-            terminateLater = True
-            processingStatus = "done"
-
         #######
         # @help
         #    get help
@@ -402,7 +301,7 @@
             h = []
             h.append("Commands:")
             h.append("  job commands:")
-            h.append("    lswjobs      get list of waiting jobs")
+            h.append("    runjob       run job on host")
             h.append("    lsrjobs      get list of running jobs")
             h.append("    lsthreads    get list of active threads")
             h.append("    lsjobinfo:<jobID>")
@@ -419,14 +318,177 @@
             h.append("                 kill all jobs matching the given string")
             h.append("")
             h.append("  server commands:")
-            h.append("    gettdinfo    get task dispatcher host and port")
-            h.append("    gettdstatus  get status of task dispatcher")
-            h.append("    gettmsinfo   get task manager server info")
-            h.append("    lsactivecluster")
-            h.append("                 show (formated) information about active cluster")
-            h.append("    terminate    termination of TMS is requested")
+            h.append("    gettmminfo   get task manager server info")
+            h.append("    shutdown     shutdown of TMMS is requested")
 
             request.send(join(h,'\n'))
+            
+        ########
+        # @check
+        #    response an ok
+        if re.match('^@check$',receivedStr):
+            request.send("I'm here.")
+            processingStatus = "done"
+
+        ########
+        # @check
+        #    response an ok
+        if re.match('^@gettmsinfo$',receivedStr):
+            try:
+                clientSock = hSocket(host=TMMS.tmsHost,
+                                     port=TMMS.tmsPort,
+                                     sslConnection=True,
+                                     certfile=certfile,
+                                     keyfile=keyfile,
+                                     ca_certs=ca_certs,
+                                     catchErrors=False)
+
+                clientSock.send("@check")
+                recv = clientSock.recv()
+                clientSock.close()
+
+                resp = []
+                resp.append("Information about connected TMS:")
+                resp.append("  Host: %s" % TMMS.tmsHost)
+                resp.append("  Port: %s" % TMMS.tmsPort)
+                if recv=="ok":
+                    resp.append("  Status: ok")
+                else:
+                    resp.append("  Status: Connection failed!")
+                request.send(join(resp,"\n"))
+            except:
+                resp = []
+                resp.append("Information about connected TMS:")
+                resp.append("  Host: %s" % TMMS.tmsHost)
+                resp.append("  Port: %s" % TMMS.tmsPort)
+                resp.append("  Status: Connection failed!")
+                request.send(join(resp,"\n"))
+            
+            processingStatus = "done"
+
+            
+        ########
+        # @runjob
+        #    run job on host and monitor
+        if re.match('^@runjob:(.*)$',receivedStr):
+            jsonInObj = re.match('@runjob:(.*)$',receivedStr).groups()[0]
+            jsonInObj = json.loads(jsonInObj)
+
+            jobID = jsonInObj['jobID']
+            command = jsonInObj['command']
+            logFile = jsonInObj.get('logFile',None)
+            shell = jsonInObj['shell']
+            
+            # create temporary file object for command which is executed
+            fCom = tempfile.NamedTemporaryFile(prefix="tmm.",bufsize=0,delete=False)
+            fCom.write("%s\n" % command)
+            fCom.close()
+
+            pid = os.getpid()
+
+            # create temporary file object for stdout and stderr of executing command
+            fOut = tempfile.NamedTemporaryFile(prefix="tmm.",bufsize=0,delete=False)
+            fErr = tempfile.NamedTemporaryFile(prefix="tmm.",bufsize=0,delete=False)
+
+            startTime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+            sp = subprocess.Popen(command,
+                                  shell=True,
+                                  executable=shell,
+                                  stdout=fOut,
+                                  stderr=fErr)
+
+
+            jsonOutObj = {'jobID': jobID,
+                          'pid': sp.pid,
+                          'fileCommand': fCom.name,
+                          'fileOutput': fOut.name,
+                          'fileError': fErr.name}
+
+            jsonOutObj = json.dumps(jsonOutObj)
+
+
+
+            # send info to TMS
+            try:
+                clientSock = hSocket(host=TMMS.tmsHost,
+                                     port=TMMS.tmsPort,
+                                     sslConnection=True,
+                                     certfile=certfile,
+                                     keyfile=keyfile,
+                                     ca_certs=ca_certs,
+                                     catchErrors=False)
+
+                clientSock.send("ProcessStarted:%s" % jsonOutObj)
+                clientSock.close()
+            except:
+                print sys.exc_info()[1]
+                sys.stderr.write("TMMS: Connection to TMS failed. Job %s is terminated." % jobID)
+                # write to log file
+                #self.killJob()
+
+            # wait until process has finished
+            sp.wait()
+
+            endTime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+            # write command, stdout, and stderr to a logfile
+            if logFile:
+                f = open('%s' % logFile,'a')
+
+                f.write("--------------------\n")
+                f.write("-----command--------\n")
+                f.write("--------------------\n")
+                f.write("%s\n" %command)
+                f.write("\n")
+                f.write("[%s] started on %s\n" % (startTime,os.uname()[1]))
+                f.write("\n")
+                f.write("[%s] finished\n" % (endTime))
+                f.write("\n")
+
+                f.write("----------------------\n")
+                f.write("-----BEGIN stdout-----\n")
+                f.write("----------------------\n")
+
+                fOut.seek(0)
+                f.write(fOut.readlines())
+
+                f.write("----------------------\n")
+                f.write("-----END stdout-------\n")
+                f.write("----------------------\n")
+                f.write("\n")
+                f.write("----------------------\n")
+                f.write("-----BEGIN stderr-----\n")
+                f.write("----------------------\n")
+
+                fErr.seek(0)
+                f.write(fErr.readlines())
+
+                f.write("----------------------\n")
+                f.write("-----END stderr-------\n")
+                f.write("----------------------\n")
+
+                f.close()
+
+            fOut.close()
+            fErr.close()
+
+            try:
+                clientSock = hSocket(host=TMMS.tmsHost,
+                                     port=TMMS.tmsPort,
+                                     sslConnection=True,
+                                     certfile=certfile,
+                                     keyfile=keyfile,
+                                     ca_certs=ca_certs,
+                                     catchErrors=False)
+
+                clientSock.send("ProcessFinished:%s:%s:%s" % (jobID,sp.pid,sp.returncode))
+                clientSock.close()
+            except:
+                sys.stderr.write("TMMS: Connection to TMS failed. ProcessFinish could not be send for job %s!" % jobID)
+                # write to log file
+            
+
+            processingStatus = "done"
             
 
         ########
@@ -436,398 +498,22 @@
             request.send(join(map(lambda t: t.getName(),threading.enumerate()),"\n"))
             
         ########
-        # @setpersistent
-        #    set socket connection persistent, i.e. do not close socket
-        elif re.match('^@setpersistent$',receivedStr):
-            request.send("set connection as persistent")
-            persistentSocket = True
-
-        ########
-        # @unsetpersistent
-        #    set socket connection persistent, i.e. do not close socket
-        elif re.match('^@unsetpersistent$',receivedStr):
-            request.send("set connection as unpersistent")
-            persistentSocket = False
-
-        ########
-        # @gettdinfo
-        #    get task dispatcher host and port
-        elif re.match('^@gettdinfo$',receivedStr):
-            request.send("%s:%s" % (TDHost,TDPort))
+        # @gettmminfo
+        #    get task manager menial server info
+        elif re.match('^@gettmminfo$',receivedStr):
+            resp = []
+            resp.append("Information about TMMS:")
+            resp.append("  Host: %s" % TMMS.host)
+            resp.append("  Port: %s" % TMMS.port)
+            resp.append("  User: %s" % TMMS.user)
+            resp.append("  Start time: %s" % TMMS.startTime)
+            resp.append("  Number of threads: %s" % threading.activeCount())
+            resp.append("  ")
+            request.send(join(resp,"\n"))
+
             processingStatus = "done"
-
-        ########
-        # @gettdstatus
-        #    get status of task dispatcher
-        elif re.match('^@gettdstatus$',receivedStr):
-            com = "@gettdstatus"
-            tdConn = TMConnection(TDHost,
-                                  TDPort,
-                                  sslConnection=TMS.sslConnection,
-                                  keyfile=keyfile,
-                                  certfile=certfile,
-                                  ca_certs=ca_certs,
-                                  catchErrors=False,
-                                  logFile=TMS.logFileTMS)
-
-            if tdConn.openConnection:
-                tdConn.sendAndRecvAndClose(com)
-                if tdConn.requestSent:
-                    request.send(tdConn.response)
-                else:
-                    request.send("Failed")
-            else:
-                request.send("Failed")
-                
-            processingStatus = "done"
-        
-        ########
-        # @gettmsinfo
-        #    get task manager server info
-        elif re.match('^@gettmsinfo$',receivedStr):
-            request.send("%s:%s:%s" % (TMS.startTime,
-                                       len(TMS.runningJobs),
-                                       (len(TMS.jobsDict)-len(TMS.runningJobs))))
-
-            processingStatus = "done"
-            
-        
-        ########
-        # @lsactivecluster
-        #    show (formated) information about active cluster
-        elif re.match('^@lsactivecluster$',receivedStr):
-            com = "@lsactivecluster"
-            tdConn = TMConnection(TDHost,
-                                  TDPort,
-                                  sslConnection=TMS.sslConnection,
-                                  keyfile=keyfile,
-                                  certfile=certfile,
-                                  ca_certs=ca_certs,
-                                  catchErrors=False,
-                                  logFile=TMS.logFileTMS)
-            
-            if tdConn.openConnection:
-                tdConn.sendAndRecvAndClose(com)
-                if tdConn.requestSent:
-                    request.send(tdConn.response)
-                else:
-                    request.send("no info")
-            else:
-                request.send("Connection to TD failed")
-                
-            processingStatus = "done"
-        
-
-
-        ########
-        # @getstatus
-        #    get status of task dispatcher and task manager server
-        #elif re.match('^@getstatus$',receivedStr):
-        #     com = "nonsense"
-        #     TDConn = TDConnection(com,
-        #                       TDHost,
-        #                       TDPort,
-        #                       receive=True,
-        #                       logFileTMS=None)
-        #                       #logFileTMS=self.logFileTMS)
-        #     TDConn.sendCommand()
-        #     if (TDConn.comSent):
-        #         request.send("%s:%s:OK:%s:%s" % (TDHost,
-        #                                          TDPort,
-        #                                          TMS.startTime,
-        #                                          len(TMS.runningJobs),
-        #                                          (len(TMS.jobs)-len(TMS.runningJobs))))
-        #     else:
-        #         request.send("%s:%s:Failed::0:0" % (TDHost, TDPort))
-
-        
-        ########
-        # @getjobstatus:<jobID>
-        #    get status of job with given jobID
-        elif re.match('@getjobstatus:(.*)',receivedStr):
-            jobID = re.match('@getjobstatus:(.*)',receivedStr).groups()[0]
-            if jobID in TMS.jobsDict:
-                request.send("Job is still waiting or running.")
-            else:
-                request.send("Unknown or finished job.")
-
-            processingStatus = "done"
-
-        ########
-        # @lswjobs
-        #    get running jobs
-        elif re.match('@lswjobs',receivedStr):
-            allJobs = set(TMS.jobsDict.keys())
-            rJobs = set(TMS.runningJobs)
-            wJobs = join(list(allJobs - rJobs),"\n")
-            if wJobs:
-                request.send(wJobs)
-            else:
-                request.send("no waiting jobs")
-
-            processingStatus = "done"
-            
-        ########
-        # @lsrjobs
-        #    get running jobs
-        elif re.match('@lsrjobs',receivedStr):
-            rJobs = join(TMS.runningJobs,",")
-            if rJobs:
-                request.send(rJobs)
-            else:
-                request.send("no running jobs")
-
-
-        ########
-        # @lsjobinfo:<jobID>
-        #    get job info of job with given jobID
-        elif re.match('@lsjobinfo:(.*)',receivedStr):
-            jobID = re.match('@lsjobinfo:(.*)',receivedStr).groups()[0]
-            com = '@lsjobinfo:%s' % jobID
-            tdConn = TMConnection(TaskDispatcherHost,
-                                  TaskDispatcherPort,
-                                  sslConnection=TMS.sslConnection,
-                                  keyfile=keyfile,
-                                  certfile=certfile,
-                                  ca_certs=ca_certs,
-                                  catchErrors=False,
-                                  logFile=TMS.logFileTMS)
-
-            if tdConn.openConnection:
-                tdConn.sendAndRecvAndClose(com)
-                if tdConn.requestSent:
-                    if tdConn.response:
-                        request.send(tdConn.response)
-                    else:
-                        request.send("Unknown job!")
-                else:
-                    request.send("Sending to TD failed")
-            else:
-                request.send("Connection to TD failed")
-
-
-        ########
-        # @getmatchingjobs:<matchString>
-        #    get all jobs with match the search string
-        elif re.match('@getmatchingjobs:(.*)',receivedStr):
-            matchString = re.match('@getmatchingjobs:(.*)',receivedStr).groups()[0]
-            com = '@lsmatchingjobs:all:%s' % matchString
-            tdConn = TMConnection(TDHost,
-                                  TDPort,
-                                  sslConnection=TMS.sslConnection,
-                                  keyfile=keyfile,
-                                  certfile=certfile,
-                                  ca_certs=ca_certs,
-                                  catchErrors=False,
-                                  logFile=TMS.logFileTMS)
-
-            if tdConn.openConnection:
-                tdConn.sendAndRecvAndClose(com)
-                if tdConn.requestSent:
-                    if TDConn.response:
-                        request.send(tdConn.response)
-                    else:
-                        request.send("Unknown job!")
-                else:
-                    request.send("Sending to TD failed")
-            else:
-                request.send("Connection to TD failed")
-
+            
                     
-        ########
-        # @addjob:<infoText>:<command>:<logFile>:<shell>:<priority>
-        #    add job to task dispatcher
-        elif re.match('@addjob:([^:]*):([^:]*):([^:]*):([^:]*):([^:]*)',receivedStr):
-            jobInfo,command,logFile,shell,priority = re.match('@addjob:([^:]*):([^:]*):([^:]*):([^:]*):([^:]*)',receivedStr).groups()
-
-            # set jobID
-            #jobID = '%s.%08d' % (TMS.info['user'],TMS.jID)
-            # jobID will now be generated by TD
-            jobID = TMS.info['user']
-
-            #self.broadcastString('[%s:%s] ... sending job %s to TD' % (h,p,jobID),TMS)
-            self.broadcastString('[%s:%s] ... sending job to TD' % (h,p),TMS)
-            self.logOutput('[%s:%s] ... sending job to TD' % (h,p),TMS)
-
-            # send command to TaskDispatcher
-            if (not priority):
-                com = "@addjob:%s:%s:%s:%s:%s:%s" % (TMS.info['host'],
-                                                     TMS.info['port'],
-                                                     TMS.ID,
-                                                     jobInfo,
-                                                     command,
-                                                     jobID)
-            else:
-                com = "@addjob:%s:%s:%s:%s:%s:%s:%s" %(TMS.info['host'],
-                                                       TMS.info['port'],
-                                                       TMS.ID,
-                                                       jobInfo,
-                                                       command,
-                                                       jobID,
-                                                       priority)
-
-            # send command to task dispatcher
-            tdConn = TMConnection(TDHost,
-                                  TDPort,
-                                  sslConnection=TMS.sslConnection,
-                                  keyfile=keyfile,
-                                  certfile=certfile,
-                                  ca_certs=ca_certs,
-                                  catchErrors=False,
-                                  logFile=TMS.logFileTMS)
-
-            if tdConn.openConnection:
-                tdConn.sendAndRecvAndClose(com)
-                if tdConn.requestSent:
-                    jobID = tdConn.response
-
-                    # add job
-                    TMS.jobsDict[jobID] = Job()
-
-                    TMS.jobsDict[jobID].setJobInfo(
-                        command = command,
-                        shell = shell,
-                        TaskDispatcherHost = TDHost,
-                        TaskDispatcherPort = TDPort
-                        )
-                    TMS.jobsDict[jobID].setAsAdded()
-
-                    if logFile:
-                        TMS.logFiles[jobID] = logFile
-
-                    self.broadcastString("[%s:%s] ... ... assigned jobID: %s" % (h,p,jobID), TMS)
-                    self.logOutput("[%s:%s] ... assigned jobID: %s" % (h,p,jobID), TMS)
-
-                    request.send("Job [%s] has been submitted to TaskDispatcher %s:%s.\nSo long, and thanks for all the fish." % (jobID,TDHost,TDPort))
-
-                    self.executeJobEvent(request,h,p,jobID,TMS)
-                else:
-                    request.send("TMS error while sending to TD %s:%s" % (TDHost,TDPort))
-                    self.broadcastString("[%s:%s] ... sent failed" % (h,p), TMS)
-                    self.logOutput("[%s:%s] ... sent failed" % (h,p), TMS)
-            else:
-                request.send("TMS error while connecting to TD %s:%s" % (TDHost,TDPort))
-                self.broadcastString("[%s:%s] ... sent failed" % (h,p), TMS)
-                self.logOutput("[%s:%s] ... sent failed" % (h,p), TMS)
-                
-
-        ########
-        # @addjobJSON:<jsonObj>     <infoText>:<command>:<logFile>:<shell>:<priority>
-        #    add job to task dispatcher
-        elif re.match('@addjobJSON:(.*)',receivedStr):
-            jsonInObj = re.match('@addjobJSON:(.*)',receivedStr).groups()[0]
-            jsonInObj = json.loads(jsonInObj)
-
-            jobInfo = jsonInObj['jobInfo']
-            command = jsonInObj['command']
-            logFile = jsonInObj['logFile']
-            shell = jsonInObj['shell']
-            priority = jsonInObj['priority']
-            excludedHosts = jsonInObj.get("excludedHosts",[])
-
-            # set jobID
-            #jobID = '%s.%08d' % (TMS.info['user'],TMS.jID)
-            # jobID will now be generated by TD
-            user = TMS.info['user']
-
-            #self.broadcastString('[%s:%s] ... sending job %s to TD' % (h,p,jobID),TMS)
-            self.broadcastString('[%s:%s] ... sending job to TD' % (h,p),TMS)
-            self.logOutput('[%s:%s] ... sending job to TD' % (h,p),TMS)
-
-            # send command to TaskDispatcher
-            jsonOutObj =  { 'TMSHost': TMS.info['host'],
-                            'TMSPort': TMS.info['port'],
-                            'TMSID': TMS.ID,
-                            'jobInfo': jobInfo,
-                            'command': command,
-                            'user': user,
-                            'priority': priority,
-                            'excludedHosts': excludedHosts}
-
-            jsonOutObj = json.dumps(jsonOutObj)
-            com = "@addjobJSON:%s" % jsonOutObj
-
-            # send command to task dispatcher
-            tdConn = TMConnection(TDHost,
-                                  TDPort,
-                                  sslConnection=TMS.sslConnection,
-                                  keyfile=keyfile,
-                                  certfile=certfile,
-                                  ca_certs=ca_certs,
-                                  catchErrors=False,
-                                  logFile=TMS.logFileTMS)
-
-            if tdConn.openConnection:
-                tdConn.sendAndRecvAndClose(com)
-                if tdConn.requestSent:
-                    jobID = tdConn.response
-
-                    # add job
-                    TMS.jobsDict[jobID] = Job()
-
-                    TMS.jobsDict[jobID].setJobInfo(
-                        command = command,
-                        shell = shell,
-                        TaskDispatcherHost = TDHost,
-                        TaskDispatcherPort = TDPort
-                        )
-                    TMS.jobsDict[jobID].setAsAdded()
-
-                    if logFile:
-                        TMS.logFiles[jobID] = logFile
-
-                    self.broadcastString("[%s:%s] ... ... assigned jobID: %s" % (h,p,jobID), TMS)
-                    self.logOutput("[%s:%s] ... assigned jobID: %s" % (h,p,jobID), TMS)
-
-                    request.send("Job (id: %s) submitted to TaskDispatcher %s:%s.\nSo long, and thanks for all the fish." % (jobID,TDHost,TDPort))
-
-                    self.executeJobEvent(request,h,p,jobID,TMS)
-                else:
-                    request.send("TMS error while sending to TD %s:%s" % (TDHost,TDPort))
-                    self.broadcastString("[%s:%s] ... sent failed" % (h,p), TMS)
-                    self.logOutput("[%s:%s] ... sent failed" % (h,p), TMS)
-            else:
-                request.send("TMS error while connecting to TD %s:%s" % (TDHost,TDPort))
-                self.broadcastString("[%s:%s] ... sent failed" % (h,p), TMS)
-                self.logOutput("[%s:%s] ... sent failed" % (h,p), TMS)
-
-
-        ########
-        # @startjob:<jobID>:<hostName>
-        #    request from task dispatcher to start a job on the given host
-        elif re.match('@startjob:(.*):(.*)',receivedStr ):
-            jobID,host = re.match('@startjob:(.*):(.*)',receivedStr ).groups()
-            
-            jobStatus = TMS.checkJobID(jobID)
-            if jobStatus == 0:
-                # job does not come from this TMS
-                self.broadcastString("[%s:%s] ... ... unknown job: %s" % (h,p,jobID), TMS)
-                self.logOutput("[%s:%s] ... unknwon job: %s" % (h,p,jobID), TMS)
-
-                # send request to td to remove this job
-                com = "@killjob:%s" % jobID
-                tdConn = TMConnection(TDHost,
-                                      TDPort,
-                                      sslConnection=TMS.sslConnection,
-                                      keyfile=keyfile,
-                                      certfile=certfile,
-                                      ca_certs=ca_certs,
-                                      catchErrors=True,
-                                      logFile=TMS.logFileTMS)
-                tdConn.sendAndClose(com)
-                
-                return persistentSocket,terminateLater
-            elif jobStatus == 1:
-                # job comes from this TMS but is unknwon
-                TMS.jobsDict[jobID] = Job()
-
-            TMS.jobsDict[jobID].setJobInfo(
-                host = host )
-
-            TMS.jobsDict[jobID].setAsStartRequested()
-            self.executeJobEvent(request,h,p,jobID,TMS)
-
-
         ########
         # ????? only jobID
         # @suspendjob:<hostName>:<pid>
@@ -863,12 +549,12 @@
             com = '@killjob:%s' % jobID
             tdConn = TMConnection(TDHost,
                                   TDPort,
-                                  sslConnection=TMS.sslConnection,
+                                  sslConnection=TMMS.sslConnection,
                                   keyfile=keyfile,
                                   certfile=certfile,
                                   ca_certs=ca_certs,
                                   catchErrors=False,
-                                  logFile=TMS.logFileTMS)
+                                  logFile=TMMS.logFileTMMS)
             tdConn.sendAndClose(com)
             
             request.send("Killing job...")
@@ -891,106 +577,24 @@
             
             tdConn = TMConnection(TaskDispatcherHost,
                                   TaskDispatcherPort,
-                                  sslConnection=TMS.sslConnection,
+                                  sslConnection=TMMS.sslConnection,
                                   keyfile=keyfile,
                                   certfile=certfile,
                                   ca_certs=ca_certs,
                                   catchErrors=False,
-                                  logFile=TMS.logFileTMS)
+                                  logFile=TMMS.logFileTMMS)
             
             tdConn.sendAndClose(com)
             
             request.send("Killing jobs.")
 
 
-        ########
-        # ProcessStarted:<jobID>:<pid>
-        #    info from hRun about a started job
-        #elif re.match('ProcessStarted:([^:]*):([^:]*)',receivedStr):
-        elif re.match('ProcessStarted:(.*)',receivedStr):
-            #jobID,pid = re.match('ProcessStarted:([^:]*):([^:]*)',receivedStr).groups()
-            jsonInObj = re.match('ProcessStarted:(.*)',receivedStr).groups()[0]
-            jsonInObj = json.loads(jsonInObj)
-
-            jobID = jsonInObj["jobID"]
-            pid = jsonInObj["pid"]
-            fileCommand = jsonInObj['fileCommand']
-            fileOutput = jsonInObj['fileOutput']
-            fileError = jsonInObj['fileError']
-            
-            jobStatus = TMS.checkJobID(jobID)
-            if jobStatus == 0:
-                # job does not come from this TMS
-                self.broadcastString("[%s:%s] ... ... unknown job: %s" % (h,p,jobID), TMS)
-                self.logOutput("[%s:%s] ... unknwon job: %s" % (h,p,jobID), TMS)
-                
-                # send request to td to remove this job
-                com = "@killjob:%s" % jobID
-                
-                tdConn = TMConnection(TDHost,
-                                      TDPort,
-                                      sslConnection=TMS.sslConnection,
-                                      keyfile=keyfile,
-                                      certfile=certfile,
-                                      ca_certs=ca_certs,
-                                      catchErrors=True,
-                                      logFile=TMS.logFileTMS)
-                
-                tdConn.sendAndClose(com)
-                
-                return persistentSocket,terminateLater
-            elif jobStatus == 1:
-                # job comes from this TMS but is unknwon
-                TMS.jobsDict[jobID] = Job()
-
-            TMS.jobsDict[jobID].setJobInfo(
-                pid = pid
-                )
-
-            TMS.jobsDict[jobID].setAsStarted()
-            self.executeJobEvent(request,h,p,jobID,TMS)
-
-
-        ########
-        # ProcessFinished:<jobID>:<pid>:<returnCode>
-        #    info from hRun about a finished job
-        elif re.match('ProcessFinished:([^:]*):([^:]*):([^:]*)',receivedStr):
-            jobID,pid,returncode = re.match('ProcessFinished:([^:]*):([^:]*):([^:]*)',receivedStr).groups()
-
-            jobStatus = TMS.checkJobID(jobID)
-            if jobStatus == 0:
-                # job does not come from this TMS
-                self.broadcastString("[%s:%s] ... ... unknown job: %s" % (h,p,jobID), TMS)
-                self.logOutput("[%s:%s] ... unknwon job: %s" % (h,p,jobID), TMS)
-
-                # send request to td to remove this job
-                com = "@killjob:%s" % jobID
-                
-                tdConn = TMConnection(TDHost,
-                                      TDPort,
-                                      sslConnection=TMS.sslConnection,
-                                      keyfile=keyfile,
-                                      certfile=certfile,
-                                      ca_certs=ca_certs,
-                                      catchErrors=True,
-                                      logFile=TMS.logFileTMS)
-                
-                tdConn.sendAndClose(com)
-                
-                return persistentSocket,terminateLater
-            elif jobStatus == 1:
-                # job comes from this TMS but is unknwon
-                TMS.jobsDict[jobID] = Job()
-
-            TMS.jobsDict[jobID].setJobInfo(returnCode=returncode)
-            TMS.jobsDict[jobID].setAsFinished()
-            self.executeJobEvent(request,h,p,jobID,TMS)
 
         ########
         # @addInfoSocket
         #    add a special socket
         elif re.match('^@addInfoSocket$',receivedStr):
-            TMS.infoSockets.append(deepcopy(request.socket))
+            TMMS.infoSockets.append(deepcopy(request.socket))
 
         ########
         # @sleep
@@ -1000,301 +604,36 @@
             self.broadcastString("... done sleeping",TMS)
 
         ########
-        # @terminate
+        # @shutdown
         #    termination is requested
-        elif re.match('^@terminate$',receivedStr):
-            # send deleting request to taskdispatcher
-            jobIDs = TMS.jobsDict.keys()
-            com = '@deletejobs:%s' % join(jobIDs,":")
-            
-            tdConn = TMConnection(TDHost,
-                                  TDPort,
-                                  sslConnection=TMS.sslConnection,
-                                  keyfile=keyfile,
-                                  certfile=certfile,
-                                  ca_certs=ca_certs,
-                                  catchErrors=True,
-                                  logFile=TMS.logFileTMS)
-            
-            tdConn.sendAndClose(com)
-
-            self.terminateAllJobs(TMS)
-            
-            request.send("terminating TMS ...")
-            TMS.doNotTerminate = False
-
-
-            #print "terminating ..."
-            #self.TMS.shutdown()
-            #print "... done terminating"
+        elif re.match('^@shutdown$',receivedStr):
+            # kill all running processes and shutdown server
+            TMMS.shutdown()
+            pass
 
         ########
         # unknown command
         else:
             #if self.verboseMode:
             #    sys.stdout.write("TMS: What do you want?\n")
-            request.send("TMS: What do you want?")
-            
-
-
-        return persistentSocket,terminateLater
-
-    def broadcastString(self,s, TMS):
-        if TMS.verboseMode:
+            request.send("TMMS: What do you want?")
+            
+
+        return
+        #return persistentSocket,terminateLater
+
+    def broadcastString(self,s, TMMS):
+        if TMMS.verboseMode:
             sys.stdout.write("[%s] %s\n" %  (datetime.now().strftime("%Y.%m.%d %H:%M:%S"),s) )
             sys.stdout.flush()
-
-        disconnectedIS = []
-        for iSocket in TMS.infoSockets:
-            try:
-                iStr = "TMS: ["+datetime.now().strftime("%Y-%m-%d %H:%M:%S")+"] "+s+"\n"
-                iSocket.send(iStr)
-            except:
-                disconnectedIS.append(iSocket)
-
-        # remove disconnectedIS
-        for iSocket in disconnectedIS:
-            TMS.infoSockets.remove(iSocket)
-            
-    def logOutput(self,s, TMS):
-        if TMS.logFileTMS:
-            TMS.logFileTMS.write("[%s] %s\n" %  (datetime.now().strftime("%Y.%m.%d %H:%M:%S"),s) )
-            TMS.logFileTMS.flush()
-
-
-    def executeJobEvent(self,request,h,p,jobID,TMS):
-        # execute job events while considering correct order of sending events to taskdispatcher
-
-        status = TMS.jobsDict[jobID].getStatus()
-        #self.broadcastString("[%s:%s] ... jobID: %s status: %s" % (h,p,jobID,status),TMS)
-        
-        if status == 3:
-            # job has already been added
-            # --> sent job to host
-
-            computerInfo = [TMS.jobsDict[jobID].getJobInfo('host')]
-
-            self.broadcastString("[%s:%s] [%s] sending to host" % (h,p,jobID), TMS)
-            self.logOutput("[%s:%s] [%s] sending to host" % (h,p,jobID), TMS)
-
-            command = TMS.jobsDict[jobID].getJobInfo('command')
-            shell = TMS.jobsDict[jobID].getJobInfo('shell')
-
-            rt = self.startJob(jobID,computerInfo,command,shell)
-
-            #############emulating running job
-            #sys.stdout.write("emulating running job %s ...\n" % jobID)
-            #sys.stdout.write("... start\n")
-            #sys.stdout.flush()
-            #
-            #h = TMS.host
-            #p = TMS.port
-            #
-            #com = "ProcessStarted:%s:001" % (jobID)
-            #TDConn = TDConnection(com,
-            #                  h,
-            #                  p,
-            #                  sslConnection=TMS.sslConnection,
-            #                  receive=False,
-            #                  catchErrors=False,
-            #                  logFileTMS=TMS.logFileTMS)
-            #TDConn.sendCommand()
-            #rt = 0
-            ##############
-            
-            
-
-            if rt:
-                # start of job failed
-                TaskDispatcherHost = TMS.jobsDict[jobID].getJobInfo('TaskDispatcherHost')
-                TaskDispatcherPort = TMS.jobsDict[jobID].getJobInfo('TaskDispatcherPort')
-
-                com = 'ProcessStartFailed:%s:%s' % (jobID,rt)
-                
-                tdConn = TMConnection(TaskDispatcherHost,
-                                      TaskDispatcherPort,
-                                      sslConnection=TMS.sslConnection,
-                                      keyfile=keyfile,
-                                      certfile=certfile,
-                                      ca_certs=ca_certs,
-                                      catchErrors=False,
-                                      logFile=TMS.logFileTMS)
-                
-                tdConn.sendAndClose(com)
-
-                self.broadcastString("[%s:%s] [%s] ... failed [%s]" % (h,p,jobID,rt),TMS)
-                self.logOutput("[%s:%s] [%s] ... failed [%s]" % (h,p,jobID,rt),TMS)
-
-                # set job status to 1 (same as added)
-                TMS.jobsDict[jobID].setAsStartFailed()
-
-            else:
-                self.broadcastString("[%s:%s] [%s] ... successfully sent to host" % (h,p,jobID),TMS)
-                self.logOutput("[%s:%s] [%s] ... successfully sent to host" % (h,p,jobID),TMS)
-
-            #############emulating running job
-            #sleep(5)
-            #sys.stdout.write("... stop\n")
-            #sys.stdout.flush()
-            #
-            #com = "ProcessFinished:%s:001:0" % (jobID)
-            #TDConn = TDConnection(com,
-            #                  h,
-            #                  p,
-            #                  sslConnection=TMS.sslConnection,
-            #                  receive=False,
-            #                  catchErrors=False,
-            #                  logFileTMS=TMS.logFileTMS)
-            #TDConn.sendCommand()
-            #############
-
-        elif status == 7:
-            # job has already been added, sent to host, and started on host
-            # --> send info to taskdispatcher
-            
-            TaskDispatcherHost = TMS.jobsDict[jobID].getJobInfo('TaskDispatcherHost')
-            TaskDispatcherPort = TMS.jobsDict[jobID].getJobInfo('TaskDispatcherPort')
-            pid = TMS.jobsDict[jobID].getJobInfo('pid')
-            fileCommand = TMS.jobsDict[jobID].getJobInfo('fileCommand')
-            fileOutput = TMS.jobsDict[jobID].getJobInfo('fileOutput')
-            fileError = TMS.jobsDict[jobID].getJobInfo('fileError')
-            
-            jsonObj = {'jobID': jobID,
-                       'pid': pid,
-                       'fileCommand': fileCommand,
-                       'fileOutput': fileOutput,
-                       'fileError': fileError}
-            
-            jsonObj = json.dumps(jsonObj)
-            
-            #com = "ProcessStarted:%s:%s" % (jobID,pid)
-            com = "ProcessStarted:%s" % (jsonObj)
-
-            tdConn = TMConnection(TaskDispatcherHost,
-                                  TaskDispatcherPort,
-                                  sslConnection=TMS.sslConnection,
-                                  keyfile=keyfile,
-                                  certfile=certfile,
-                                  ca_certs=ca_certs,
-                                  catchErrors=False,
-                                  logFile=TMS.logFileTMS)
-            
-            tdConn.sendAndClose(com)
-
-            TMS.runningJobs.append(jobID)
-
-            self.broadcastString("[%s:%s] [%s] started on host" % (h,p,jobID),TMS)
-            self.logOutput("[%s:%s] [%s] started on host" % (h,p,jobID),TMS)
-
-        elif status == 15:
-            # job has already been added, sent to host, started, and finished
-            # --> send info to taskdispatcher
-            
-            TaskDispatcherHost = TMS.jobsDict[jobID].getJobInfo('TaskDispatcherHost')
-            TaskDispatcherPort = TMS.jobsDict[jobID].getJobInfo('TaskDispatcherPort')
-            returnCode = TMS.jobsDict[jobID].getJobInfo('returnCode')
-            
-            com = "ProcessFinished:%s:%s" % (jobID, returnCode)
-
-            tdConn = TMConnection(TaskDispatcherHost,
-                                  TaskDispatcherPort,
-                                  sslConnection=TMS.sslConnection,
-                                  keyfile=keyfile,
-                                  certfile=certfile,
-                                  ca_certs=ca_certs,
-                                  catchErrors=False,
-                                  logFile=TMS.logFileTMS)
-            
-            tdConn.sendAndClose(com)
-
-            try:
-                TMS.runningJobs.remove(jobID)
-            except:
-                pass
-
-            try:
-                del TMS.jobsDict[jobID]
-            except:
-                pass
-
-            self.broadcastString("[%s:%s] [%s] finished" % (h,p,jobID),TMS)
-            self.logOutput("[%s:%s] [%s] finished" % (h,p,jobID),TMS)
-
-
-
-    def startJob(self,jobID,computerInfo,command,shell):
-        computer = computerInfo[0]
-        command = command.replace("'","\"'\"")
-        
-        if (jobID in TMS.logFiles):
-            logFile = TMS.logFiles[jobID]
-            job = """{binPath}/hRun -i {jobID} -l {logFile} -S {shell} {host} {port} \"{command}\"; """.format(**{'binPath':binPath,
-                                                                                                           'jobID':jobID,
-                                                                                                           'port':TMS.info['port'],
-                                                                                                           'host':TMS.info['host'],
-                                                                                                           'logFile':logFile,
-                                                                                                           'shell':shell,
-                                                                                                           'command':command
-                                                                                                           })
-            del TMS.logFiles[jobID]
-        else:
-            #job = """%s/hRun -i %s -p %s -s %s -S %s \"%s\"; """ % (binPath,
-            #                                                              jobID,
-            #                                                              TMS.info['port'],
-            #                                                              TMS.info['host'],
-            #                                                              shell,
-            #                                                              command)
-            job = """{binPath}/hRun -i {jobID} -S {shell} {host} {port} \"{command}\"; """.format(**{'binPath':binPath,
-                                                                                              'jobID':jobID,
-                                                                                              'port':TMS.info['port'],
-                                                                                              'host':TMS.info['host'],
-                                                                                              'shell':shell,
-                                                                                              'command':command
-                                                                                              })
-        # problem with shell command substitution:
-        # for instance:
-        #     hRunJob.py "echo 17 | sed 's/^.*$/42/' > /home/hache/tmp/out.txt"
-        # gives error "Illegal variable name.", since the shell tries to substitute $ by a variable.
-        # Therefore $ has to be somehow escaped:
-        #     hRunJob.py "echo 17 | sed 's/^.*"'$'"/42/' > /home/hache/tmp/out.txt"
-        # 
-        # further: suprocess performes apparently as well variable substitution. therefore, e.g.,
-        # $ has to be replaced by "'$'". The same has to be done for !.
-        # -------> solved with substitution of ' to"'"
-        
-        #job = join(re.split(r'(?<=[^$])\$(?=[^$])', job),"\"'$'\"")
-        #job = join(re.split(r'(?<=[^!])!(?=[^!])', job),"\"'!'\"")
-
-        #self.broadcastString("Complete job: %s" % job)
-    
-        #subprocess.Popen('"%s"' % job,shell=True, executable='/bin/tcsh')
-        #sp = subprocess.Popen(job,shell=True)
-
-        if TMS.logFileTMS:
-            TMS.logFileTMS.write("["+datetime.now().strftime("%Y.%m.%d %H:%M:%S")+"] Run: "+"ssh -x -a "+computer+" "+job+"\n")
-            TMS.logFileTMS.flush()
-
-
-        # start job
-        try:
-            sp = subprocess.Popen(['ssh', '-x', '-a', computer, job])
-        except:
-            # an error occured
-            sys.stderr.write("[%s] TMS: Error while invoking ssh connection to %s\n" % (datetime.now().strftime("%Y.%m.%d %H:%M:%S"),computer) )
-            sys.stderr.write("--> ERROR: %s\n" % sys.exc_info()[1])
-            sys.stderr.flush()
-            if TMS.logFileTMS:
-                TMS.logFileTMS.write("["+datetime.now().strftime("%Y.%m.%d %H:%M:%S")+"] Error while invoking ssh connection to "+computer+"!\n")
-                TMS.logFileTMS.flush()
-            return sys.exc_info()[1]
-
-        #if TMS.logFileTMS:
-        #    TMS.logFileTMS.write("["+datetime.now().strftime("%Y.%m.%d %H:%M:%S")+"] %s sp2 returncode %s\n" % (jobID,sp2.returncode))
-        #    TMS.logFileTMS.flush()
-        
-        return sp.poll()
-    
-        
+            
+    def logOutput(self,s, TMMS):
+        if TMMS.logFileTMMS:
+            TMMS.logFileTMMS.write("[%s] %s\n" %  (datetime.now().strftime("%Y.%m.%d %H:%M:%S"),s) )
+            TMMS.logFileTMMS.flush()
+
+
+    # ???
     def suspendJob(self,computer,pid):
         if pid:
             job="ssh -x -a %s pstree -p %s" % (computer,pid)
@@ -1309,6 +648,7 @@
                     subprocess.Popen(j1,shell=True)
                 if not line: break
 
+    # ???
     def resumeJob(self,computer,pid):
         if pid:
             job="ssh -x -a %s pstree -p %s" % (computer,pid)
@@ -1323,75 +663,30 @@
                     subprocess.Popen(j1,shell=True)
                 if not line: break
 
+    # ???
     def terminateJob(self,jobID,TMS):
-        if jobID not in TMS.jobsDict:
-            self.broadcastString("unkown job to kill: %s" % (jobID),TMS)
-            return
-        
-        host = TMS.jobsDict[jobID].getJobInfo('host')
-        pid = TMS.jobsDict[jobID].getJobInfo('pid')
-
-        if pid:
-            job="ssh -x -a %s pstree -p %s" % (host,pid)
-            self.broadcastString("host: %s, pid: %s" % (host,pid),TMS)
-            self.logOutput("host: %s, pid: %s" % (host,pid),TMS)
-            rePIDS=re.compile('\(\d+\)')
-            output=os.popen(job)
-            self.broadcastString("pstree output: %s" % (output),TMS)
-            self.logOutput("pstree output: %s" % (output),TMS)
-            while 1:
-                line = output.readline()
-                for m in re.finditer(rePIDS,line):
-                    pid1=m.group().replace('(','')
-                    pid1=pid1.replace(')','')
-                    j1="ssh -x -a %s kill -9 %s" %(host,pid1)
-                    self.broadcastString("kill command: %s" % (j1),TMS)
-                    self.logOutput("kill command: %s" % (j1),TMS)
-                    subprocess.Popen(j1,shell=True)
-                if not line: break
-        
-        # remove job from internal lists
-        try:
-            del TMS.jobsDict[jobID]
-            #TMS.jobs.remove(jobID)
-
-            #jobPrIDs= [jobID,pid]
-            #self.runningJobs.remove(jobPrIDs)
-            self.jobs.remove(jobID)
-            self.runningJobs.remove(jobID)
-        except:
-            pass
-
-        
-    def terminateAllJobs(self,TMS):
-        jobIDs = TMS.jobsDict.keys()
-        for jobID in jobIDs:
-            computer = TMS.jobsDict[jobID].getJobInfo('host')
-            self.terminateJob(jobID,TMS)            
-
-        TMS.jobsDict = {}
-
-
-
+        pass
+
+    # ???
+    def terminateAllJobs(self,TMMS):
+        pass
+
+    # ???
     def killSuspendedJob(self,jobID):
-        try:
-            del TMS.jobsDict[jobID]
-            self.jobs.remove(jobID)
-        except:
-            pass
+        pass
+    
 
 if __name__ == '__main__':
     try:
-        opts,args = getopt.getopt(sys.argv[1:],"hdl:tT:v")
+        opts,args = getopt.getopt(sys.argv[1:],"hdl:T:v")
     except getopt.error, message:
         sys.stderr.write('%s: Error!! %s\n' % (sys.argv[0].split('/')[-1],message) )
         printHelp(sys.stderr)
         sys.exit(-1)
 
     runNotAsDaemon = False
-    doNotTerminate = False
     verboseMode = False
-    logFileTMS = None
+    logFileTMMS = None
     for option, param in opts:
         if option == '-h':
             printHelp(sys.stdout)
@@ -1400,15 +695,11 @@
             runNotAsDaemon = True
         elif option == "-l":
             try:
-                logFileTMS = open(param,'w')
+                logFileTMMS = open(param,'w')
             except IOError,msg:
-                sys.stderr.write("TMS ERROR: %s\n" % msg)
-                sys.stderr.write("TMS: logfile is ignored\n")
-
-        #elif option == "-T":
-        #    TaskDispatcherHost,TaskDispatcherPort = param.split(":")
-        elif option == "-t":
-            doNotTerminate = True
+                sys.stderr.write("TMMS ERROR: %s\n" % msg)
+                sys.stderr.write("TMMS: logfile is ignored\n")
+
         elif option == '-v':
             verboseMode = True
 
@@ -1416,48 +707,53 @@
         sys.stderr.write('%s: Error!! %s\n' % (sys.argv[0].split('/')[-1],"Port number is not given!") )
         printHelp(sys.stderr)
         sys.exit(-1)
-    elif len(args)>1:
-        sys.stderr.write('%s: Error!! %s\n' % (sys.argv[0].split('/')[-1],"Too many arguments!") )
+    elif len(args)<3:
+        sys.stderr.write('%s: Error!! Arguments are missing!\n' % (sys.argv[0].split('/')[-1]) )
         printHelp(sys.stderr)
         sys.exit(-1)
+    elif len(args)>3:
+        sys.stderr.write('%s: Error!! Too many arguments!\n' % (sys.argv[0].split('/')[-1]) )
+        printHelp(sys.stderr)
+        sys.exit(-1)
 
     try:
+        tmsHost = args[1]
+        tmsPort = int(args[2])
+
         port = int(args[0])
         host= uname()[1]
-        user=pwd.getpwuid(os.getuid())[0]
-
-        TMS=TaskManagerServer(user=user,
-                              host=host,
-                              port=port,
-                              sslConnection=True,
-                              keyfile=keyfile,
-                              certfile=certfile,
-                              ca_certs=ca_certs,
-                              handler=TaskManagerServerHandler,
-                              processor=TaskManagerServerProcessor(),
-                              verboseMode=verboseMode,
-                              doNotTerminate=doNotTerminate,
-                              logFileTMS=logFileTMS)
+
+        TMMS=TaskManagerMenialServer(host=host,
+                                    port=port,
+                                    tmsHost=tmsHost,
+                                    tmsPort=tmsPort,
+                                    sslConnection=True,
+                                    keyfile=keyfile,
+                                    certfile=certfile,
+                                    ca_certs=ca_certs,
+                                    handler=TaskManagerMenialServerHandler,
+                                    processor=TaskManagerMenialServerProcessor(),
+                                    verboseMode=verboseMode,
+                                    #doNotTerminate=doNotTerminate,
+                                    logFileTMMS=logFileTMMS)
 
         if runNotAsDaemon:
-            TMS.run() # run not as daemen; for debugging
-            #TMS.sendCommandToTMSM("@terminate",expectResponse=False)
+            TMMS.run() # run not as daemen; for debugging
         else:
-            TMS.start() # run as deamon
-            #TMS.sendCommandToTMSM("@terminate",expectResponse=False)
+            TMMS.start() # run as deamon
 
     except exceptions.KeyboardInterrupt:
-        sys.stderr.write("TMS KeyboardInterrupt")
-        sys.stderr.write("TMS terminated.\n")
+        sys.stderr.write("TMMS KeyboardInterrupt")
+        sys.stderr.write("TMMS terminated.\n")
     except socket.error,msg:
-        sys.stderr.write("TMS (%s:%s) Socket Error: %s\n" % (host,port,msg))
-        sys.stderr.write("TMS terminated.\n")
+        sys.stderr.write("TMMS (%s:%s) Socket Error: %s\n" % (host,port,msg))
+        sys.stderr.write("TMMS terminated.\n")
     except SystemExit,msg:
-        sys.stderr.write("TMS System Error: %s\n" % msg)
-        sys.stderr.write("TMS terminated.\n")
+        sys.stderr.write("TMMS System Error: %s\n" % msg)
+        sys.stderr.write("TMMS terminated.\n")
     except:
-        sys.stderr.write("TMS exception.")
-        sys.stderr.write("TMS Error: %s\n" % sys.exc_info()[1])
+        sys.stderr.write("TMMS exception.")
+        sys.stderr.write("TMMS Error: %s\n" % sys.exc_info()[1])
         sys.stderr.write("TRACEBACK:")
         traceback.print_exc(file=sys.stderr)
         #raise