diff -Nur Webware.ori/WebKit/AppServer Webware.new/WebKit/AppServer --- Webware.ori/WebKit/AppServer Tue Sep 12 14:32:56 2000 +++ Webware.new/WebKit/AppServer Thu Nov 23 15:19:20 2000 @@ -1,2 +1,2 @@ -python ThreadedAppServer.py +python ThreadedAppServer.py $* diff -Nur Webware.ori/WebKit/AppServer.py Webware.new/WebKit/AppServer.py --- Webware.ori/WebKit/AppServer.py Thu Nov 30 14:57:49 2000 +++ Webware.new/WebKit/AppServer.py Thu Nov 30 15:27:48 2000 @@ -24,6 +24,9 @@ 'Webware.PSP', 'Webware.WebKit', 'Webware.WebUtils'], + 'Host': '127.0.0.1', + 'Port': 8086, + 'ServerThreads': 10, 'ApplicationClassName': 'Webware.WebKit.Application.Application' } diff -Nur Webware.ori/WebKit/Application.py Webware.new/WebKit/Application.py --- Webware.ori/WebKit/Application.py Thu Nov 30 14:59:10 2000 +++ Webware.new/WebKit/Application.py Tue Nov 21 19:17:15 2000 @@ -17,8 +17,6 @@ from Cookie import Cookie from Webware.WebUtils.WebFuncs import HTMLEncode -from Webware.WebUtils.HTMLForException import HTMLForException - class ApplicationError(Exception): @@ -270,7 +268,7 @@ def dispatchRawRequest(self, newRequestDict): return self.dispatchRequest(self.createRequestForDict(newRequestDict)) - def dispatchRequest(self, request): + def dispatchRequest(self, request, oldresponse=None): """ Creates the transaction, session, response and servlet for the new request which is then dispatched. The transaction is returned. """ assert request is not None @@ -281,6 +279,7 @@ transaction = self.createTransactionForRequest(request) response = self.createResponseInTransaction(transaction) + if oldresponse: response.setParrentResponse(oldresponse) try: ssPath = request.serverSidePath() @@ -298,15 +297,13 @@
%s
''' % sys.stdout.getvalue()) sys.stdout = real_stdout - response.deliver(transaction) - # (*) We have to use pathInfo() instead of uri() when looking for the trailing slash, because some webservers, notably Apache, append a trailing / to REQUEST_URI in some circumstances even though the user did not specify that (for example: http://localhost/WebKit.cgi). except: if transaction: transaction.setErrorOccurred(1) self.handleExceptionInTransaction(sys.exc_info(), transaction) - transaction.response().deliver(transaction) # I hope this doesn't throw an exception. :-) @@ 2000-05-09 ce: provide a secondary exception handling mechanism + # I hope this doesn't throw an exception. :-) @@ 2000-05-09 ce: provide a secondary exception handling mechanism pass if self.setting('LogActivity'): @@ -410,8 +407,9 @@ newRequest = self.createRequestForDict(req.rawRequest()) newRequest.setURLPath(urlPath) - newTrans = self.dispatchRequest(newRequest) - trans.response().appendRawResponse(newTrans.response().rawResponse()) + newTrans = self.dispatchRequest(newRequest, trans.response()) + if not newTrans.response().isCommitted(): + trans.response().appendResponse(newTrans.response()) def forwardRequestFast(self, trans, URL): @@ -609,7 +607,8 @@ return self._requestClass(dict=newRequestDict) def createTransactionForRequest(self, request): - trans = self._transactionClass(application=self, request=request) + trans = self._transactionClass(application=self, + request=request, file=request.output()) request.setTransaction(trans) return trans @@ -786,7 +785,7 @@ if debug: print '>> serverSidePathForRequest(request=%s)' % repr(request) import pprint - pprint.pprint(request._rawRequest) + pprint.pprint(request.rawRequest()) urlPath = request.urlPath() if debug: print '>> urlPath =', repr(urlPath) @@ -948,16 +947,9 @@ """ Returns a raw reponse. This method is mostly used by OneShotAdaptor.py. """ - from Webware.WebUtils.HTMLForException import HTMLForException - try: - assert type(requestDict) is type({}) - app = Application(useSessionSweeper=0) - return app.dispatchRawRequest(requestDict).response().rawResponse() - except: - return { - 'headers': [('Content-type', 'text/html')], - 'contents': '%s' % HTMLForException() - } + assert type(requestDict) is type({}) + app = Application(useSessionSweeper=0) + return app.dispatchRawRequest(requestDict).deliver() # You can run Application as a main script, in which case it expects a single diff -Nur Webware.ori/WebKit/AsyncThreadedAppServer.py Webware.new/WebKit/AsyncThreadedAppServer.py --- Webware.ori/WebKit/AsyncThreadedAppServer.py Thu Nov 30 14:59:10 2000 +++ Webware.new/WebKit/AsyncThreadedAppServer.py Tue Nov 21 20:28:45 2000 @@ -6,21 +6,36 @@ from AppServer import AppServer from Application import Application -from marshal import dumps, loads +try: + from cPickle import dump, load, dumps +except ImportError: + from pickle import dump, load, dumps + import os, sys -from threading import Lock, Thread +from threading import Thread, Condition import Queue import select import socket import asyncore from Webware.WebUtils.WebFuncs import RequestURI +import string, time +from types import DictType +import traceback + try: from selectRelease import Release - MainRelease=Release() + MainRelease=Release() # what it's it ? except: MainRelease = lambda:() +AddToDefaultConfig = { + 'AsyncoreBufferSize' : 1024 * 1024 # one mega for one request + } + + + +# A little test suite class AsyncThreadedAppServer(asyncore.dispatcher, AppServer): """ @@ -30,6 +45,7 @@ self._addr = None self._poolsize=self.setting('ServerThreads') + self._buffersize=self.setting('AsyncoreBufferSize',1024*1024) self.threadPool=[] self.requestQueue=Queue.Queue(self._poolsize*5) #5 times the number of threads we have @@ -46,6 +62,7 @@ self.running=1 out = sys.stdout + out.write('Buffer size %d\n' % self._buffersize) out.write('Creating %d threads' % self._poolsize) for i in range(self._poolsize): #change to threadcount t = Thread(target=self.threadloop) @@ -60,7 +77,14 @@ self.listen(64) # @@ 2000-07-14 ce: hard coded constant should be a setting print "Ready\n" + out.flush() + + ## Configuration ## + def defaultConfig(self): + x=AppServer.defaultConfig(self).copy() + x.update(AddToDefaultConfig) + return x def isPersistent(self): return 1 @@ -108,12 +132,9 @@ def address(self): if self._addr is None: - self._addr = (self.setting('Host'), self.setting('Port')) + self._addr = (self.setting('Host'),self.setting('Port')) return self._addr - - - def threadloop(self): """ Try to get a RequestHandler instance off the requestQueue and process it. @@ -155,9 +176,236 @@ del self._app +class AsyncRead: + # methods to simulating read file-access to ayncore.dispatcher, + # idea from StringIO + + def __init__(self, dispatcher, buffersize): + self._dispatcher=dispatcher + self._buffersize=buffersize + self.reset() + + def reset(self): + self.len = 0 + self.buflist = [] + self.closed = 0 + self.softspace = 0 + self._hasMore=1 # in socket + self._notQueued=1 # in request queue + def allReaded(self): return not self._hasMore -import string, time + def canRecv(self): + return self._hasMore and self._notQueued + + def recv(self): + assert self._hasMore and self._notQueued + + data = self._dispatcher.recv(8192) + lendata=len(data) + self.len=self.len+lendata + + if lendata == 8192: + self.buflist.append(data) + else: + self.buflist.append(data) + self._hasMore=0 # all is readed + + if self._hasMore and self.lenn: break + outlist.append(data) + if outlen>n: + diff=n-outlen+lendata + outlist.append(data[:diff]) + self.buflist.insert(0, data[diff:]) + elif self._hasMore and outlen=length: + i=length + else: + outlist.append(data) + return (0,length and length-len(data)) + elif length and i>=length: i=length + else: i=i+1 + + dd=data[:i] + outlist.append(dd) + self.buflist.insert(0,data[i:]) + return (1,length and length-len(data)) # read all + + + def readline(self, length=None): + assert not self._notQueued + if self.closed: + raise ValueError, "I/O operation on closed file" + + if length==0: return '' + elif self.buflist: + outlist=[] + while self.buflist: + data=self.buflist.pop(0) + con,length = self._readlinedata( + data,length,outlist) + if con: break + if self._hasMore and not con: + outlist.append(self.readline(length)) + return string.joinfields(outlist,'') + elif self._hasMore: + sock=self._dispatcher.socket + try : + outlist=[] + sock.setblocking(1) + while 1: + if self.buflist:data=self.buflist.pop(0) + else: + data=sock.recv(8192) + if not data: + self._hasMore=0 + break + con,length = self._readlinedata( + data,length,outlist) + if con: break + return string.joinfields(outlist,'') + finally: + sock.setblocking(0) + else: return '' + + def readlines(self): + lines = [] + line = self.readline() + while line: + lines.append(line) + line = self.readline() + return lines + + def flush(self): + if self.closed: + raise ValueError, "I/O operation on closed file" +class AsyncWrite: + # methods to simulating write file-access to asyncore.dispatcher, + # idea from StringIO + + def __init__(self, dispatcher, buffersize): + self._dispatcher=dispatcher + self._buffersize=buffersize + self._bufflock = Condition() + self.reset() + + def reset(self): + self.len=0 + self.buflist = [] + self.closed = 0 + self.softspace = 0 + + def canSend(self): + return self.buflist + + def send(self): + assert self.buflist + + self._bufflock.acquire() + try: + # join all data + data=string.joinfields(self.buflist,'') + sent = self._dispatcher.send(data) + if sent==len(data): self.len,self.buflist=0,[] + else: + a=data[sent:] + self.buflist=[a] + self.len=len(a) + + if self.closed: # all is write to buffer + if not self.buflist: # + self._dispatcher.close() + else: self._bufflock.notify() + finally: + self._bufflock.release() + + # file access + def close(self): + if not self.closed: + self.closed = 1 + + def isatty(self): + if self.closed: + raise ValueError, "I/O operation on closed file" + return 0 + + def write(self, s): + if self.closed: + raise ValueError, "I/O operation on closed file" + if not s: return + self._bufflock.acquire() + try: + while self.len>self._buffersize: + self._bufflock.wait() + self.len=self.len+len(s) + self.buflist.append(s) + finally: + self._bufflock.release() + + def writelines(self, list): + self.write(string.joinfields(list, '')) + + def flush(self): + if self.closed: + raise ValueError, "I/O operation on closed file" class RequestHandler(asyncore.dispatcher): """ @@ -168,17 +416,11 @@ def __init__(self, server): self.server=server - self.have_request = 0 - self.have_response = 0 - self.reqdata=[] - self._buffer = '' - + self.readfile=AsyncRead(self,self.server._buffersize) + self.writefile=AsyncWrite(self,self.server._buffersize) + def handleRequest(self): #check for status message - if self.reqdata == "STATUS": - self._buffer = str(self.server._reqCount) - self.have_response = 1 - return verbose = self.server._verbose @@ -186,39 +428,33 @@ if verbose: print 'BEGIN REQUEST' print time.asctime(time.localtime(startTime)) - - if verbose: print 'receiving request from', self.socket - while not self.have_request: - print ">> Should always have request before we get here" - time.sleep(0.01) - + dict=load(self.readfile) + assert type(dict)==DictType if verbose: - print 'received %d bytes' % len(self.reqdata) - if self.reqdata: - dict = loads(self.reqdata) - if verbose: - print 'request has keys:', string.join(dict.keys(), ', ') - if dict.has_key('environ'): - requestURI = RequestURI(dict['environ']) - else: - requestURI = None - print 'request uri =', requestURI - else: dict={} - - if dict.get('format')=='AUTH': - if verbose: - print "AUTH", dict.get('REMOTE_USER') + print 'request has keys:', string.join(dict.keys(), ', ') + if dict.has_key('environ'): + requestURI = RequestURI(dict['environ']) + else: + requestURI = None + print 'request uri =', requestURI + format=dict.get('format') + if format == "STATUS": + if verbose: print "STATUS", self.server._reqCount + self.writefile.write(str(self.server._reqCount)) + elif format=='AUTH': + if verbose: print "AUTH", dict.get('REMOTE_USER') transaction=None - rawResponse=dumps( - self.server._app.dispatchAuthRequest(dict)) + self.writefile.write(dumps( + self.server._app.dispatchAuthRequest(dict),1)) else: - transaction=self.server._app.dispatchRawRequest(dict) - rawResponse=dumps(transaction.response().rawResponse()) + dict['input']=self.readfile + dict['output']=self.writefile + transaction = self.server._app.dispatchRawRequest(dict) + transaction.deliver() - self._buffer = rawResponse - self.have_response = 1 + self.writefile.close() if verbose: print 'connection closed.' @@ -234,53 +470,35 @@ def activate(self, socket): self.set_socket(socket) - self._buffer='' self.active = 1 def readable(self): - return self.active and not self.have_request + return self.active and self.readfile.canRecv() def writable(self): """ Always ready to write, otherwise we might have to wait for a timeout to be asked again """ - #return self.active - return self.have_response + return self.writefile.canSend() def handle_connect(self): pass def handle_read(self): - data = self.recv(8192) - if len(data) == 8192: - self.reqdata.append(data) - else: - self.reqdata.append(data) - self.reqdata = string.join(self.reqdata,'') - self.have_request=1 - self.server.requestQueue.put(self) - #self.socket.shutdown(0) + self.readfile.recv() def handle_write(self): - if not self.have_response: return - sent = self.send(self._buffer) - self._buffer = self._buffer[sent:] - if len(self._buffer) == 0: - self.close() - #For testing - if __debug__: - sys.stdout.write(".") - sys.stdout.flush() + self.writefile.send() def close(self): self.active = 0 - self.have_request = 0 - self.have_response = 0 - self.reqdata=[] + self.readfile.reset() + self.writefile.reset() asyncore.dispatcher.close(self) self.server.rhQueue.put(self) def log(self, message): + #print "LOG ",message pass class Monitor(asyncore.dispatcher): @@ -313,8 +531,6 @@ def log(self, message): pass - - def main(monitor = 0): try: server = None @@ -323,14 +539,8 @@ monitor = Monitor(server) asyncore.loop() except Exception, e: #Need to kill the Sweeper thread somehow - print e + traceback.print_exc() print "Exiting AppServer" - if 0: #See the traceback from an exception - tb = sys.exc_info() - print tb[0] - print tb[1] - import traceback - traceback.print_tb(tb[2]) if server: server.running=0 server.shutDown() @@ -338,10 +548,23 @@ sys.exit() raise Exception() - if __name__=='__main__': - import sys - if len(sys.argv) > 1 and sys.argv[1] == "-monitor": + import sys, getopt, os + optlist,args=getopt.getopt(sys.argv[1:],'?', + ['monitor','pidfile=','help']) + dict={} + for i,j in optlist: dict[i]=j + if dict.has_key('--help') or dict.has_key('-?'): + print """Asyncore Threaded Application server from Webware for python +Usage : python %s [--monitor] [--pidfile file] + """ % sys.argv[0] + sys.exit() + if dict.has_key('--pidfile'): + f=open(dict['--pidfile'],'w') + f.write(str(os.getpid())+'\n') + f.close() + + if dict.has_key('--monitor'): print "Using Monitor" main(1) else: diff -Nur Webware.ori/WebKit/CGIAdaptor.py Webware.new/WebKit/CGIAdaptor.py --- Webware.ori/WebKit/CGIAdaptor.py Fri Sep 29 23:00:18 2000 +++ Webware.new/WebKit/CGIAdaptor.py Tue Nov 21 19:44:24 2000 @@ -30,12 +30,14 @@ timestamp = time.time() from socket import * -from marshal import dumps, loads +try: + from cPickle import dumps, loads +except ImportError: + from pickle import dumps, loads debugging = 0 # set 1 if you want to see the raw response dictionary, instead of a normal page - def main(): import os, sys @@ -46,17 +48,10 @@ msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) - myInput = '' - if os.environ.has_key('CONTENT_LENGTH'): - length = int(os.environ['CONTENT_LENGTH']) - if length: - myInput = sys.stdin.read(length) - dict = { 'format': 'CGI', 'time': timestamp, - 'environ': os.environ.data, # ce: a little tricky. We use marshal which only works on built-in types, so we need environ's dictionary - 'input': myInput + 'environ': os.environ.data # ce: a little tricky. We use pickle which only works on built-in types, so we need environ's dictionary } # 2000-05-20 ce: For use in collecting raw request dictionaries for use in Testing/stress.py @@ -75,30 +70,40 @@ s = socket(AF_INET, SOCK_STREAM) s.connect((host, port)) - s.send(dumps(dict)) + s.send(dumps(dict,1)) + + # send raw input to socket + myInput=sys.stdin.read(bufsize) + while myInput: + s.send(myInput) + myInput=sys.stdin.read(bufsize) s.shutdown(1) - # read the raw reponse (which is a marshalled dictionary) - response = '' - while 1: - chunk = s.recv(bufsize) - if not chunk: - break - response = response + chunk - response = loads(response) # decode it + # make file from socket + file=s.makefile("r") + # read the raw reponse, marshaled are only headers + headers=load(file) + # deliver it! write = sys.stdout.write if debugging: - write('Content-type: text/html\n\n') - write('

Your adapter has debugging set to true.

') - write(HTMLEncode(str(response))) - write('') - else: - for pair in response['headers']: - write('%s: %s\n' % pair) - write('\n') - write(response['contents']) + sys.stdout.write( + 'Content-type: text/html\n\n') + sys.stdout.write( + '

Your adapter has debugging set to true.

') + write=lambda x : sys.stdout.write(HTMLEncode(x)) + + for pair in headers: + write('%s: %s\n' % pair) + write('\n') + + # copy response to output + response=file.read(bufsize) + while response: + write(response) + response=file.read(bufsize) + if debugging: sys.stdout.write('') except: import traceback diff -Nur Webware.ori/WebKit/Examples/Colorize.py Webware.new/WebKit/Examples/Colorize.py --- Webware.ori/WebKit/Examples/Colorize.py Thu Nov 30 14:57:49 2000 +++ Webware.new/WebKit/Examples/Colorize.py Tue Nov 21 20:37:42 2000 @@ -21,10 +21,15 @@ res.write("No filename given to syntax color!") return #filename = req.relativePath(req.field('filename')+'.py') - filename = req.field('filename') + filename = os.path.normpath(req.field('filename')) if not os.path.exists(filename): res.write(filename+" does not exist.") return + elif filename[:19]!='/usr/lib/python1.5/' or \ + filename[:25]!='/usr/local/lib/python1.5/': + res.write("No no no !!! :-b") + return + from Webware.WebKit.DocSupport import py2html diff -Nur Webware.ori/WebKit/ExceptionHandler.py Webware.new/WebKit/ExceptionHandler.py --- Webware.ori/WebKit/ExceptionHandler.py Thu Nov 30 14:57:49 2000 +++ Webware.new/WebKit/ExceptionHandler.py Tue Nov 7 18:57:39 2000 @@ -66,7 +66,7 @@ self._res.recordEndTime() self.logExceptionToConsole() - self._res.reset() + if not self._res.isCommitted(): self._res.reset() if self.setting('ShowDebugInfoOnErrors')==1: publicErrorPage = self.privateErrorPage() else: diff -Nur Webware.ori/WebKit/FCGIAdaptor.py Webware.new/WebKit/FCGIAdaptor.py --- Webware.ori/WebKit/FCGIAdaptor.py Fri Sep 29 23:00:18 2000 +++ Webware.new/WebKit/FCGIAdaptor.py Tue Nov 21 19:44:34 2000 @@ -73,10 +73,18 @@ * Increased rec buffer size from 8KB to 32KB * Removed use of pr() for feeding app server results back to webserver. Figure that's slightly more efficient. * Added notes about how I set this up with Apache to what was already there. + +* 2000-11-22 vlk: + * Switched from marshal.dumps() encoding to pickle.dumps() encoding in accordance with AppServer + + """ import fcgi, time -from marshal import dumps, loads +try: + from cPickle import dumps, loads +except ImportError: + from pickle import dumps, loads from socket import * import string import os @@ -117,8 +125,7 @@ dict = { 'format': 'CGI', 'time': timestamp, - 'environ': env, - 'input': form + 'environ': env } @@ -127,24 +134,31 @@ s = socket(AF_INET, SOCK_STREAM) s.connect((host, port)) - s.send(dumps(dict)) + s.send(dumps(dict,1)) + # send raw input to socket + myInput=form.read(bufsize) + while myInput: + s.send(myInput) + myInput=form.read(bufsize) s.shutdown(1) - # read the raw reponse (which is a marshalled dictionary) - response = '' - while 1: - chunk = s.recv(bufsize) - if not chunk: - break - response = response + chunk - response = loads(response) # decode ita + # make file from socket + file=s.makefile("r") + # read the raw reponse, marshaled are only headers + headers=load(file) + # deliver it! write = fcg.req.out.write - for pair in response['headers']: + for pair in headers: write('%s: %s\n' % pair) write('\n') - write(response['contents']) + + # copy response to output + response=file.read(bufsize) + while response: + write(response) + response=file.read(bufsize) fcg.req.out.flush() except: diff -Nur Webware.ori/WebKit/HTTPRequest.py Webware.new/WebKit/HTTPRequest.py --- Webware.ori/WebKit/HTTPRequest.py Thu Nov 30 14:59:10 2000 +++ Webware.new/WebKit/HTTPRequest.py Wed Nov 8 10:16:18 2000 @@ -2,7 +2,7 @@ from Request import Request from Webware.WebUtils.Cookie import Cookie import cgi -from types import ListType +from types import ListType, StringType, DictionaryType import os from Webware.WebUtils.WebFuncs import RequestURI @@ -19,14 +19,26 @@ def __init__(self, dict={}): Request.__init__(self) - self._rawRequest = dict.copy() if dict: # Dictionaries come in from web server adaptors like the CGIAdaptor assert dict['format']=='CGI' self._time = dict['time'] self._environ = dict['environ'] - self._input = dict['input'] - self._fields = cgi.FieldStorage(fp=StringIO(self._input), environ=self._environ, keep_blank_values=1, strict_parsing=0) + self._input = dict['input'] + self._output = dict['output'] + if type(self._input)==StringType: + self._fields = cgi.FieldStorage( + fp=StringIO(self._input), + environ=self._environ, + keep_blank_values=1,strict_parsing=0) + elif type(self._input)==DictionaryType: + # in forwardRequest is send as dictionary + self._fields=self._input['_fields'] + self._cookies=self._input['_cookies'] + else: self._fields = cgi.FieldStorage(fp=self._input, + environ=self._environ, + keep_blank_values=1, strict_parsing=0) + self._cookies = Cookie() if self._environ.has_key('HTTP_COOKIE'): self._cookies.load(self._environ['HTTP_COOKIE']) @@ -36,9 +48,12 @@ self._time = time.time() self._environ = os.environ.copy() self._input = '' + self._output = sys.stdout self._fields = cgi.FieldStorage(keep_blank_values=1) self._cookies = Cookie() + self._input={'_fields':self._fields, '_cookies': self._cookies } + # Debugging if 0: f = open('env.text', 'a') @@ -73,10 +88,17 @@ dict = {} for key in self._fields.keys(): value = self._fields[key] - if type(value) is not ListType: - value = value.value # i.e., if we don't have a list, we have one of those cgi.MiniFieldStorage objects. Get it's value. - else: - value = map(lambda miniFieldStorage: miniFieldStorage.value, value) # extract those .value's + if value.filename: # its a uploaded file + value=(value.filename,value.file) + elif type(value) is not ListType: + value = value.value + # i.e., if we don't have a list, we have one + # of those cgi.MiniFieldStorage objects. + # Get it's value. + # We must check if its a uploaded file + else: value = map(lambda x:\ + (x.file and (x.filename,x.file)\ + or x.value), value) dict[key] = value self._fields = dict @@ -258,11 +280,23 @@ return self._adapterName def rawRequest(self): - ''' Returns the raw request that was used to initialize this request object. ''' - return self._rawRequest + ''' Returns the raw request that as used to initialize this request object. ''' + # @@ 2000-11-06 vlk Not the same bat can be constructed + # make string from REQUEST ( env, time, .... ) + return { + 'format' : 'CGI', + 'time' : self._time, + 'environ' : self._environ, + 'input' : self._input, + 'output' : self._output + } def environ(self): return self._environ # @@ 2000-05-01 ce: To implement ExceptionHandler.py + + def output(self): + ''' File-descriptor where transaction will send output ''' + return self._output ## Information ## diff -Nur Webware.ori/WebKit/HTTPResponse.py Webware.new/WebKit/HTTPResponse.py --- Webware.ori/WebKit/HTTPResponse.py Sun Aug 27 02:04:18 2000 +++ Webware.new/WebKit/HTTPResponse.py Tue Nov 21 19:44:49 2000 @@ -1,6 +1,10 @@ from Common import * from Response import Response from Cookie import Cookie +try: + from cPickle import dump +except ImportError: + from pickle import dump class HTTPResponse(Response): @@ -21,7 +25,7 @@ self._cookies = {} self._committed = 0 self._contents = [] - + self._parrentResponse=None ## Headers ## @@ -106,21 +110,42 @@ self._contents = [ ' This page has been redirected to %s. ' % (url, url)] + def setParrentResponse(self,response): + ''' Used by Application.forwardRequest, see deliver ''' + self._parrentResponse=response ## Output ## def write(self, string): - assert self._committed==0 - self._contents.append(str(string)) + if self._committed==0: + self._contents.append(str(string)) + else: # sent string directly to file + self._file.write(string) def isCommitted(self): return self._committed def deliver(self, trans): - self.recordSession(trans) - self._contents = string.join(self._contents, '') + ''' Sent contents to file-descriptor ''' + if self._committed==0: + if self._parrentResponse: + if not self._parrentResponse.isCommitted(): + self._parrentResponse.mergeHeaders(self) + self._parrentResponse.deliver(trans) + else: + self.recordSession(trans) + self._file=trans.file() + assert self._file + headers = self._headers.items() + headers.extend(map(lambda cookie: \ + ('Set-Cookie', cookie.headerValue()), + self._cookies.values())) + dump(headers,self._file,1) + + for i in self._contents: + self._file.write(i) + self._committed = 1 self.recordEndTime() - self._committed = 1 def recordSession(self, trans): ''' Invoked by deliver() to record the session id in the response (if a session exists). This implementation sets a cookie for that purpose. For people who don't like sweets, a future version could check a setting and instead of using cookies, could parse the HTML and update all the relevant URLs to include the session id (which implies a big performance hit). Or we could require site developers to always pass their URLs through a function which adds the session id (which implies pain). Personally, I'd rather just use cookies. You can experiment with different techniques by subclassing Session and overriding this method. Just make sure Application knows which "session" class to use. ''' @@ -137,38 +162,28 @@ def reset(self): ''' Resets the response (such as headers, cookies and contents). ''' - self._committed = 0 # @@ 2000-06-09 ce: this fixes ExceptionHandler problem where it reuses the response. maybe this is "bad" + assert self._committed==0 self._headers = {'Content-type': 'text/html'} self._cookies = {} self._contents = [] - def rawResponse(self): - ''' Returns the final contents of the response. Don't invoke this method until after deliver(). - Returns a dictionary representing the response containing only strings, numbers, lists, tuples, etc. with no backreferences. That means you don't need any special imports to examine the contents and you can marshal it. Currently there are two keys. 'headers' is list of tuples each of which contains two strings: the header and it's value. 'contents' is a string (that may be binary (for example, if an image were being returned)). ''' - headers = [] - for key, value in self._headers.items(): - headers.append((key, value)) - for cookie in self._cookies.values(): - headers.append(('Set-Cookie', cookie.headerValue())) - return { - 'headers': headers, - 'contents': self._contents - } - - def size(self): - ''' Returns the size of the final contents of the response. Don't invoke this method until after deliver(). ''' - assert self._contents is not None, 'Contents are not set. Perhaps deliver() has not been invoked.' - return len(self._contents) - def appendRawResponse(self, rawRes): + def mergeHeaders(self, resp): ''' - Appends the contents of the raw response (as returned by some transaction's rawResponse() method) to this response. - The headers of the receiving response take precedence over the appended response. - This method was built primarily to support Application.forwardRequest(). + Merge headers and cookies from another response. + The headers of the receiving response take precedence over the + appended response. This method was built primarily to support + Application.forwardRequest(). ''' assert self._committed==0 - headers = rawRes.get('headers', []) - for key, value in headers: + for key, value in resp.headers().items(): if not self._headers.has_key(key): self._headers[key] = value - self.write(rawRes['contents']) + for key,value in resp.cookies().items(): + if not self._cookies.has_key(key): + self._cookies[key] = value + + def appendResponse(self,resp): + self.mergeHeaders(resp) + for i in resp._contents: + self._contents.append(i) diff -Nur Webware.ori/WebKit/OneShotAdapter.py Webware.new/WebKit/OneShotAdapter.py --- Webware.ori/WebKit/OneShotAdapter.py Mon Aug 7 23:13:51 2000 +++ Webware.new/WebKit/OneShotAdapter.py Tue Nov 21 19:15:04 2000 @@ -24,7 +24,7 @@ _real_stdout = sys.stdout sys.stdout = _console = StringIO() # to capture the console output of the application - +import tempfile import os, string @@ -45,32 +45,40 @@ def run(self): try: - myInput = sys.stdin.read() + outfile=tempfile.TemporaryFile('w+b') + if os.name=='nt': # MS Windows: no special translation of end-of-lines + import msvcrt + msvcrt.setmode(outfile.fileno(),os.O_BINARY) dict = { 'format': 'CGI', 'time': _timestamp, - 'environ': os.environ.data, # ce: a little tricky. We use marshal which only works on built-in types, so we need environ's dictionary - 'input': myInput + 'environ': os.environ.data, # ce: a little tricky. We use pickle which only works on built-in types, so we need environ's dictionary + 'input': sys.stdin, + 'output': outfile# send output to temp file } print 'ONE SHOT MODE\n' from OneShotAppServer import OneShotAppServer appSvr = OneShotAppServer() - response = appSvr.dispatchRawRequest(dict).response().rawResponse() + appSvr.dispatchRawRequest(dict).deliver() appSvr.shutDown() appSvr = None sys.stdout = _real_stdout - if os.name=='nt': # MS Windows: no special translation of end-of-lines - import msvcrt - msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) write = sys.stdout.write - for pair in response['headers']: + + outfile.seek(0) # rewind file + headers=load(outfile) # read headers + for pair in headers: write('%s: %s\n' % pair) write('\n') - write(response['contents']) + while 1: # read and write body + data=outfile.read(BUFSIZE) + if not data: break + write(data) + if self.setting('ShowConsole'): self.showConsole(_console.getvalue()) except: diff -Nur Webware.ori/WebKit/Response.py Webware.new/WebKit/Response.py --- Webware.ori/WebKit/Response.py Tue May 23 13:34:27 2000 +++ Webware.new/WebKit/Response.py Tue Nov 7 09:07:38 2000 @@ -18,8 +18,9 @@ ## Init ## - def __init__(self): + def __init__(self,file=None): Message.__init__(self) + self.file=file # @@ 2000-04-30 ce: Does a response need to know it's transaction? #self._transaction = transaction diff -Nur Webware.ori/WebKit/ThreadedAppServer.py Webware.new/WebKit/ThreadedAppServer.py --- Webware.ori/WebKit/ThreadedAppServer.py Thu Nov 30 14:59:10 2000 +++ Webware.new/WebKit/ThreadedAppServer.py Tue Nov 21 20:30:08 2000 @@ -17,13 +17,17 @@ from Common import * from AppServer import AppServer -from marshal import dumps, loads +try: + from cPickle import dumps, load +except ImportError: + from pickle import dumps, load import os, sys from threading import Lock, Thread import Queue import select import socket from Webware.WebUtils.WebFuncs import RequestURI +import traceback DefaultConfig = { @@ -75,6 +79,8 @@ out.write("\n") self.mainsocket.listen(64) # @@ 2000-07-10 ce: hard coded constant should be a setting + print 'Ready\n' + out.flush() def isPersistent(self): return 1 @@ -191,23 +197,14 @@ conn = self.sock if verbose: print 'receiving request from', conn - recv = conn.recv - BUFSIZE = 8*1024 - data = [] - while 1: - chunk = recv(BUFSIZE) - if not chunk: - break - data.append(chunk) - data = string.join(data, '') + file=conn.makefile('rb') + dict=load(file) - if data == "STATUS": + if dict.get('format') == "STATUS": conn.send(str(self.server._reqCount)) conn.close() - - class RequestHandler: def __init__(self, server): @@ -234,40 +231,27 @@ recv = conn.recv BUFSIZE = 8*1024 - data = [] - while 1: - chunk = recv(BUFSIZE) - if not chunk: - break - data.append(chunk) - data = string.join(data, '') + # make file from socket + file=conn.makefile('rb') + dict=load(file) if verbose: - print 'received %d bytes' % len(data) - if data: - dict = loads(data) - if verbose: - print 'request has keys:', string.join(dict.keys(), ', ') - if dict.has_key('environ'): - requestURI = RequestURI(dict['environ']) - else: - requestURI = None - print 'request uri =', requestURI - else: dict={} + print 'request has keys:', string.join(dict.keys(), ', ') + if dict.has_key('environ'): + requestURI = RequestURI(dict['environ']) + else: + requestURI = None + print 'request uri =', requestURI if dict.get('format')=='AUTH': transaction=None - rawResponse=dumps( - self.server._app.dispatchAuthRequest(dict)) + conn.send(dumps( + self.server._app.dispatchAuthRequest(dict),1)) else: - transaction =self.server._app.dispatchRawRequest(dict) - rawResponse =dumps(transaction.response().rawResponse()) - - - reslen = len(rawResponse) - sent = 0 - while sent < reslen: - sent = sent + conn.send(rawResponse[sent:sent+8192]) + dict['input']=file + dict['output']=conn.makefile('wb') + transaction = self.server._app.dispatchRawRequest(dict) + transaction.deliver() conn.close() @@ -332,14 +316,8 @@ monitor = Monitor(server) server.mainloop(monitor) except Exception, e: #Need to kill the Sweeper thread somehow - print e + traceback.print_exc() print "Exiting AppServer" - if 0: #See the traceback from an exception - tb = sys.exc_info() - print tb[0] - print tb[1] - import traceback - traceback.print_tb(tb[2]) if server: server.running=0 server.shutDown() @@ -348,8 +326,23 @@ if __name__=='__main__': - import sys - if len(sys.argv) > 1 and sys.argv[1] == "-monitor": + import sys, getopt, os + optlist,args=getopt.getopt(sys.argv[1:],'?', + ['monitor','pidfile=','help']) + dict={} + for i,j in optlist: dict[i]=j + print dict + if dict.has_key('--help') or dict.has_key('-?'): + print """Threaded Application server from Webware for python +Usage : python %s [--monitor] [--pidfile file] + """ % sys.argv[0] + sys.exit() + if dict.has_key('--pidfile'): + f=open(dict['--pidfile'],'w') + f.write(str(os.getpid())+'\n') + f.close() + + if dict.has_key('--monitor'): print "Using Monitor" main(1) else: diff -Nur Webware.ori/WebKit/Transaction.py Webware.new/WebKit/Transaction.py --- Webware.ori/WebKit/Transaction.py Mon Aug 14 20:55:07 2000 +++ Webware.new/WebKit/Transaction.py Tue Nov 7 14:55:30 2000 @@ -22,7 +22,7 @@ ## Init ## - def __init__(self, application, request=None): + def __init__(self, application, request=None, file=None): Object.__init__(self) self._application = application self._request = request @@ -30,6 +30,8 @@ self._session = None self._servlet = None self._errorOccurred = 0 + if not file and self._request: self._file=self._request.output() + else: self._file = file ## Access ## @@ -78,6 +80,18 @@ self._errorOccurred = flag self._servlet = None + def file(self): + ''' Return file on which response can write ''' + return self._file + + def setFile(self,file): + ''' Set output file-descriptor for response ''' + assert not self.response() or self.response().isCommitted() + self._file=file + + def deliver(self): + ''' Deliver response to file ''' + return self.response().deliver(self) ## Transaction stages ## diff -Nur Webware.ori/WebKit/modpHandler.py Webware.new/WebKit/modpHandler.py --- Webware.ori/WebKit/modpHandler.py Thu Nov 30 14:59:10 2000 +++ Webware.new/WebKit/modpHandler.py Tue Nov 21 20:34:34 2000 @@ -35,10 +35,12 @@ from mod_python import apache import time +try: + from cPickle import dumps, load, loads +except ImportError: + from pickle import dumps, load, loads from socket import * -from marshal import dumps -from marshal import loads import sys import string @@ -48,7 +50,7 @@ port = None bufsize = 32*1024 -WEBWARE_ADDRESS_FILE='/usr/lib/python1.5/site-packages/Webware/WebKit/address.text' +WEBWARE_ADDRESS_FILE='/usr/local/lib/python1.5/site-packages/Webware/WebKit/address.text' if not port: @@ -65,16 +67,8 @@ try: timestamp = time.time() - myInput = "" - inp = req.read(bufsize) - # this looks like bad performance if we don't get it all - # in the first buffer - while inp: - myInput = myInput + inp - inp = req.read(bufsize) - if debug: - ot.write("*************\n%s\n************\n"% len(myInput)) + ot.write("*************\n???\n************\n") for name in req.headers_in.keys(): ot.write("%s=%s\n"%(name,req.headers_in[name])) ot.write(req.uri) @@ -95,43 +89,42 @@ dict = { 'format': 'CGI', 'time': timestamp, - 'environ': env, - 'input': myInput + 'environ': env } s = socket(AF_INET, SOCK_STREAM) s.connect((host, port)) - s.send(dumps(dict)) + s.send(dumps(dict,1)) + inp = req.read(bufsize) + while inp: + s.send(inp) + inp = req.read(bufsize) + s.shutdown(1) # # Get the response from the AppServer, extract the headers, # and send everything to Apache through mod_python's req object # - inheader=1 - header = "" - resp = '' - while 1: - data = s.recv(bufsize) - if not data: - break - resp = resp+data + # make file from socket + file=s.makefile("r") - respdict = loads(resp) - for pair in respdict['headers']: + headers = load(file) + for pair in headers: req.headers_out[pair[0]] = str(pair[1]) req.content_type = req.headers_out['content-type'] if req.headers_out.has_key('status'): req.status = int(req.headers_out['status']) req.send_http_header() - req.write(respdict['contents']) - + response=file.read(bufsize) + while response: + req.write(response) + response=file.read(bufsize) if debug: ot.write("+++\n\n") ot.close() - except: import traceback @@ -166,16 +159,9 @@ port = int(port) timestamp = time.time() - myInput = "" - inp = req.read(bufsize) - # this looks like bad performance if we don't get it all - # in the first buffer - while inp: - myInput = myInput + inp - inp = req.read(bufsize) if debug: - ot.write("*************\n%s\n************\n"% len(myInput)) + ot.write("*************\n???\n************\n") for name in req.headers_in.keys(): ot.write("%s=%s\n"%(name,req.headers_in[name])) ot.write("*************************\n") @@ -195,38 +181,38 @@ dict = { 'format': 'CGI', 'time': timestamp, - 'environ': env, - 'input': myInput + 'environ': env } s = socket(AF_INET, SOCK_STREAM) s.connect((host, port)) - s.send(dumps(dict)) + s.send(dumps(dict,1)) + # send raw input to socket + inp = req.read(bufsize) + while inp: + s.send(inp) + inp = req.read(bufsize) s.shutdown(1) # # Get the response from the AppServer, extract the headers, # and send everything to Apache through mod_python's req object # - inheader=1 - header = "" - resp = '' - while 1: - data = s.recv(bufsize) - if not data: - break - resp = resp+data + # make file from socket + file=s.makefile("r") - respdict = loads(resp) - for pair in respdict['headers']: - req.headers_out[pair[0]] = str(pair[1]) + headers = load(file) + for key,value in headers: + req.headers_out[key] = str(value) req.content_type = req.headers_out['content-type'] if req.headers_out.has_key('status'): req.status = int(req.headers_out['status']) req.send_http_header() - req.write(respdict['contents']) - + response=file.read(bufsize) + while response: + req.write(response) + response=file.read(bufsize) if debug: ot.write("+++\n\n") @@ -317,7 +303,7 @@ s = socket(AF_INET, SOCK_STREAM) s.connect((host, port)) - s.send(dumps(dict)) + s.send(dumps(dict,1)) s.shutdown(1) data=loads(s.recv(bufsize)) # only one integer diff -Nur Webware.ori/WebKit/wkMonitor.py Webware.new/WebKit/wkMonitor.py --- Webware.ori/WebKit/wkMonitor.py Mon Sep 4 15:39:56 2000 +++ Webware.new/WebKit/wkMonitor.py Tue Nov 21 20:45:05 2000 @@ -41,6 +41,10 @@ import sys import signal import string +try: + from cPickle import dumps +except ImportError: + from pickle import dumps #Windows Specific Section @@ -135,7 +139,7 @@ sts = time.time() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect(addr) - s.send("STATUS") + s.send(dumps({"format":"STATUS"},1)) s.shutdown(1) resp = s.recv(9)#up to 1 billion requests! monwait = time.time() - sts