Webware for Python

Update of /cvsroot/webware/Webware/WebKit
In directory usw-pr-cvs1:/tmp/cvs-serv5321/WebKit

Modified Files:
	Launch.py ServletFactory.py ThreadedAppServer.py 
Added Files:
	AutoReloadingAppServer.py ImportSpy.py 
Log Message:
Implemented auto-restart for the appserver, so that it automatically restarts
itself to pick up changes in source files.

The initial implementation came from Tavis Rudd (webware-exp), which I 
extended to add a few niceties and FAM (file alteration monitor) support.

I extended the implementation to also monitor __init__.py files and files which 
weren't imported successfully (such as modules containing syntax errors).  

This is done by using Python's ihooks.py to wrap import statements, which 
allows us to find out about imported modules, while having (hopefully) no
maintenance issues (ihooks.py is in the python distribution).

The AutoReloadingAppServer will use python-fam (python interface
to the file alteration monitor daemon) if it is installed; otherwise it
polls to check for file modifications.



--- NEW FILE: AutoReloadingAppServer.py ---
from AppServer import AppServer
import os
from threading import Thread
import time
import sys
import select

''' The purpose of this module is to notice changes to source files, including
servlets, PSPs, templates or changes to the Webware source file themselves,
and reload the server as necessary to pick up the changes.  

The server will also be restarted if a file which Webware _tried_ to import
is modified.  This is so that changes to a file containing a syntax error 
(which would have prevented it from being imported) will also cause the server 
to restart.

The server is restarted using exec, and uses the same command-line parameters
are were initially given.  Since exec replaces the existing process, there
should be no permissions issues and the process id (PID) will remain the same.
'''

# Attempt to use python-fam (fam = File Alteration Monitor) instead of polling
# to see if files have changed.  If python-fam is not installed, we fall back
# to polling.
try:
	import _fam
	haveFam = 1
except:
	haveFam = 0

from ImportSpy import modloader

DefaultConfig = {
	'AutoReload':                 0,
	'AutoReloadPollInterval':     1,  # in seconds
}

class AutoReloadingAppServer(AppServer):


	## AppServer methods which this class overrides

	def __init__(self,path=None):
		AppServer.__init__(self,path)
		self._autoReload = 0
		self._shouldRestart = 0
		self._requests = []
		self._pipe = None

		if self.isPersistent():
			# I don't think something like exec() exists in Win32, so we 
			# can only support auto-reloading under posix
			if self.setting('AutoReload') and os.name == 'posix':
				self.activateAutoReload()

	def defaultConfig(self):
		conf = AppServer.defaultConfig(self)
		conf.update(DefaultConfig)
		return conf

	def shutDown(self):
		print 'Stopping AutoReload Monitor'
		sys.stdout.flush()
		self._shuttingDown = 1
		self.deactivateAutoReload()
		AppServer.shutDown(self)


	## Activatation of AutoReload

	def activateAutoReload(self):
		"""Start the monitor thread"""
		if not self._autoReload:
			if haveFam:
				print 'AutoReload Monitor started, using FAM'
				self._fileMonitorThread = t = Thread(target=self.fileMonitorThreadLoopFAM)
			else:
				print 'AutoReload Monitor started, polling every %d seconds' % self.setting('AutoReloadPollInterval')
				self._fileMonitorThread = t = Thread(target=self.fileMonitorThreadLoop)
			self._autoReload = 1
			t.setName('AutoReloadMonitor')
			t.start()

	def deactivateAutoReload(self):
		"""Tell the monitor thread to stop."""
		self._autoReload = 0
		if haveFam:
			# send a message down the pipe to wake up the monitor
			# thread and tell him to quit
			self._pipe[1].write('close')
			self._pipe[1].flush()
		try:
			self._fileMonitorThread.join()
		except:
			pass


	## Restart methods

	def restartIfNecessary(self):
		"""This should be called regularly to see if a restart is required,
		and if so, reinitialize the process using os.execve()

		Tavis Rudd claims: "this method can only be called by the main thread.
		If a worker thread calls it, the process will freeze up."  

		I've implemented it so that the ThreadedAppServer's control thread
		calls this.  That thread is _not_ the MainThread (the initial thread 
		created by the Python interpreter), but I've never encountered any 
		problems.  Most likely Tavis meant a freeze would occur if a 
		_worker_ called this.
		"""
		if self._shouldRestart:
			self.restart()

	def restart(self):
		"""Do the actual restart."""
		self.initiateShutdown()
		self._closeThread.join()
		sys.stdout.flush()
		sys.stderr.flush()
		L = [sys.executable]
		# shutdown has already completed so 'start' not 'restart'
		L.extend([i.replace('restart','start') for i in sys.argv])
		os.execve(L[0], L, os.environ)

	
	## Callbacks

	def monitorNewModule(self,filepath,mtime):
		""" This is a callback which ImportSpy invokes to notify us
		of new files to monitor.  This is only used when we are using FAM."""
		self._requests.append( self._fc.monitorFile(filepath, filepath) )


	## Internal methods

	def shouldRestart(self):
		"""Tell the main thread to restart the server."""
		self._shouldRestart = 1

	def fileMonitorThreadLoop(self, getmtime=os.path.getmtime):
		pollInterval = self.setting('AutoReloadPollInterval')
		while self._autoReload:
			time.sleep(pollInterval)

			for f, mtime in modloader.fileList().items():
				try:
					if mtime < getmtime(f):
						print '*** The file', f, 'has changed.  The server is restarting now.'
						self._autoReload = 0
						return self.shouldRestart()
				except OSError:
					print '*** The file', f, 'is no longer accessible.  The server is restarting now.'
					self._autoReload = 0
					return self.shouldRestart()
		print 'Autoreload Monitor stopped'
		sys.stdout.flush()

	def fileMonitorThreadLoopFAM(self, getmtime=os.path.getmtime):
		modloader.notifyOfNewFiles(self.monitorNewModule)
		self._fc = fc = _fam.open()

		# for all of the modules which have _already_ been loaded, we check
		# to see if they've already been modified or deleted
		for f, mtime in modloader.fileList().items():
			if mtime < getmtime(f):
				try:
					if mtime < getmtime(f):
						print '*** The file', f, 'has changed.  The server is restarting now.'
						self._autoReload = 0
						return self.shouldRestart()
				except OSError:
					print '*** The file', f, 'is no longer accessible  The server is restarting now.'
					self._autoReload = 0
					return self.shouldRestart()
			# request that this file be monitored for changes
			self._requests.append( fc.monitorFile(f, f) )

		# create a pipe so that this thread can be notified when the
		# server is shutdown.  We use a pipe because it needs to be an object
		# which will wake up the call to 'select'
		r,w = os.pipe()
		r = os.fdopen(r,'r')
		w = os.fdopen(w,'w')
		self._pipe = pipe = (r,w)
		while self._autoReload:
			try:
				# we block here until a file has been changed, or until
				# we receive word that we should shutdown (via the pipe)
				ri, ro, re = select.select([fc,pipe[0]], [], [])
			except select.error, er:
				errnumber, strerr = er
				if errnumber == errno.EINTR:
					continue
				else:
					print strerr
					sys.exit(1)
			while fc.pending():
				fe = fc.nextEvent()
				if fe.code2str() in ['changed','deleted','created']:
					print '*** The file %s has changed.  The server is restarting now.' % fe.userData
					self._autoReload = 0
					return self.shouldRestart()
		for req in self._requests:
			req.cancelMonitor()
		fc.close()
		print 'Autoreload Monitor stopped'
		sys.stdout.flush()

--- NEW FILE: ImportSpy.py ---
import ihooks
import os

''' ImportSpy.py

The purpose of this module is to record the filepath of every module which 
is imported.  This is used by the AutoReloadingAppServer (see doc strings 
for more information) to restart the server if any source files change.

Other than keeping track of the filepaths, the behaviour of this module
loader is identical to Python's default behaviour.

'''

class ModuleLoader(ihooks.ModuleLoader):

	def __init__(self):
		ihooks.ModuleLoader.__init__(self)
		self._fileList = {}
		self._notifyHook = None

	def load_module(self,name,stuff):
		try:
			mod = ihooks.ModuleLoader.load_module(self,name,stuff)
			self.recordFileName(stuff,mod)
		except:
			self.recordFileName(stuff,None)
			raise
		return mod

	def fileList(self):
		return self._fileList

	def notifyOfNewFiles(self,hook):
		''' Called by someone else to register that they'd like to 
		be know when a new file is imported '''
		self._notifyHook = hook

	def addToFileList(self,filepath,getmtime=os.path.getmtime):
		modtime = getmtime(filepath)
		self._fileList[filepath] = modtime
		# send notification that this file was imported 
		if self._notifyHook:
			self._notifyHook(filepath,modtime)

	def recordFileName(self,stuff,mod,isfile=os.path.isfile):
		file,pathname,desc = stuff

		fileList = self._fileList
		if mod:
			# __orig_file__ is used for cheetah and psp mods; we want 
			# to record the source filenames, not the auto-generated modules
			f2 = getattr(mod, '__orig_file__', 0) 
			f = getattr(mod, '__file__', 0)

			if f2 and f2 not in fileList.keys():
				try:
					if isfile(f2):
						self.addToFileList(f2)
				except OSError:
					pass
			elif f and f not in fileList.keys():
				# record the .py file corresponding to each '.pyc'
				if f[-4:] == '.pyc':
					f = f[:-1]
				try:
					if isfile(f):
						self.addToFileList(f)
					else:
						self.addToFileList(os.path.join(f, '__init__.py'))
				except OSError:
					pass

		# also record filepaths which weren't successfully loaded, which
		# may happen due to a syntax error in a servlet, because we also want
		# to know when such a file is modified
		elif pathname:
			if isfile(pathname):
				self.addToFileList(pathname)
			else:
				print 'LALA: ' + pathname

# install our custom ModuleLoader.  Any subsequently-imported modules will
# be noticed.
modloader = ModuleLoader()
imp = ihooks.ModuleImporter(loader=modloader)
ihooks.install(imp)

''' These two methods are compatible with the 'imp' module (and can
therefore be useds as drop-in replacements), but will use the
above ModuleLoader to record the pathnames of imported modules.
'''

def load_module(name, file, filename, description):
	return modloader.load_module(name,(file,filename,description))

def find_module(name,path=None):
	return modloader.find_module(name,path)

Index: Launch.py
===================================================================
RCS file: /cvsroot/webware/Webware/WebKit/Launch.py,v
retrieving revision 1.12
retrieving revision 1.13
diff -C2 -d -r1.12 -r1.13
*** Launch.py	6 Aug 2001 18:55:30 -0000	1.12
--- Launch.py	24 Oct 2002 22:24:18 -0000	1.13
***************
*** 1,4 ****
--- 1,5 ----
  #!/usr/bin/env python
  
+ import ImportSpy
  import os, sys, string
  

Index: ServletFactory.py
===================================================================
RCS file: /cvsroot/webware/Webware/WebKit/ServletFactory.py,v
retrieving revision 1.23
retrieving revision 1.24
diff -C2 -d -r1.23 -r1.24
*** ServletFactory.py	14 Jul 2002 17:06:49 -0000	1.23
--- ServletFactory.py	24 Oct 2002 22:24:18 -0000	1.24
***************
*** 3,7 ****
  import sys
  from types import ClassType
! import imp
  import threading
  
--- 3,7 ----
  import sys
  from types import ClassType
! import ImportSpy as imp   # ImportSpy provides find_module and load_module
  import threading
  

Index: ThreadedAppServer.py
===================================================================
RCS file: /cvsroot/webware/Webware/WebKit/ThreadedAppServer.py,v
retrieving revision 1.57
retrieving revision 1.58
diff -C2 -d -r1.57 -r1.58
*** ThreadedAppServer.py	17 Jul 2002 15:22:47 -0000	1.57
--- ThreadedAppServer.py	24 Oct 2002 22:24:18 -0000	1.58
***************
*** 17,21 ****
  
  from Common import *
! from AppServer import AppServer
  from MiscUtils.Funcs import timestamp
  from marshal import dumps, loads
--- 17,21 ----
  
  from Common import *
! from AutoReloadingAppServer import AutoReloadingAppServer as AppServer
  from MiscUtils.Funcs import timestamp
  from marshal import dumps, loads
***************
*** 170,173 ****
--- 170,175 ----
  				self.manageThreadCount()
  			else: threadCheck = threadCheck+1
+ 
+ 			self.restartIfNecessary()
  
  	def activeThreadCount(self):