Update of /cvsroot/webware/Webware/WebKit
In directory usw-pr-cvs1:/tmp/cvs-serv18052/WebKit
Modified Files:
SessionDynamicStore.py
Log Message:
Rewriting SessionDynamicStore for improved efficiency and thread-safety. Also removed some unused 'smart interval' code which was cluttering up the source. If anyone ever wants to resurrect and finish the smart interval code, it's still in CVS...
Index: SessionDynamicStore.py
===================================================================
RCS file: /cvsroot/webware/Webware/WebKit/SessionDynamicStore.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** SessionDynamicStore.py 2001/02/25 18:35:37 1.7
--- SessionDynamicStore.py 2001/12/18 13:45:39 1.8
***************
*** 1,5 ****
from SessionStore import SessionStore
import SessionMemoryStore, SessionFileStore
! import os, time
debug = 0
--- 1,5 ----
from SessionStore import SessionStore
import SessionMemoryStore, SessionFileStore
! import os, time, threading
debug = 0
***************
*** 13,24 ****
To use this Session Store, set SessionStore in Application.config to 'Dynamic'.
Other variables which can be set in Application.config are:
!
'MaxDynamicMemorySessions', which sets the maximum number of sessions that can be
in the Memory SessionStore at one time. Default is 10,000.
!
'DynamicSessionTimeout', which sets the default time for a session to stay in memory
with no activity. Default is 15 minutes. When specifying this in Application.config, use minutes.
-
-
"""
--- 13,22 ----
To use this Session Store, set SessionStore in Application.config to 'Dynamic'.
Other variables which can be set in Application.config are:
!
'MaxDynamicMemorySessions', which sets the maximum number of sessions that can be
in the Memory SessionStore at one time. Default is 10,000.
!
'DynamicSessionTimeout', which sets the default time for a session to stay in memory
with no activity. Default is 15 minutes. When specifying this in Application.config, use minutes.
"""
***************
*** 27,104 ****
def __init__(self, app):
SessionStore.__init__(self, app)
self._fileStore = SessionFileStore.SessionFileStore(app)
self._memoryStore = SessionMemoryStore.SessionMemoryStore(app, restoreFiles=0)
self._memoryStore.clear() #fileStore will have the files on disk
-
- #Should we adapt the move to file interval with the load on the server?
- self.intelligentSessionArchive = self.application().setting("IntelligentSessionArchive",0)
-
- #moveToFileInterval specifies after what period of time a session is automatically moved to file
- self.moveToFileInterval = self.application().setting('DynamicSessionTimeout', 15) * 60
! #minMemoryInterval is the minimum amount of time Sessions will stay in memory, used in interval seeking
! self.minMemoryInterval = 300 #5 minutes
!
! #maxMemoryInterval is the maximum amount of time Sessions will stay in memory, used in interval seeking
! self.maxMemoryInterval = self.moveToFileInterval * 48 #12 hours by default
! #maxDynamicMemorySessions is what the user actually sets in Application.config
self._maxDynamicMemorySessions = self.application().setting('MaxDynamicMemorySessions', 10000)
self._fileSweepCount = 0
if debug:
print "SessionDynamicStore Initialized"
!
## Access ##
def __len__(self):
! return len(self._memoryStore) + len(self._fileStore)
def __getitem__(self, key):
! if self._fileStore.has_key(key):
! self.MovetoMemory(key)
! #let it raise a KeyError otherwise
! return self._memoryStore[key]
!
def __setitem__(self, key, item):
self._memoryStore[key] = item
! try:
! del self._fileStore[key]
! except KeyError:
! pass
def __delitem__(self, key):
! if self._memoryStore.has_key(key):
! del self._memoryStore[key]
! if self._fileStore.has_key(key):
! del self._fileStore[key]
def has_key(self, key):
! return self._memoryStore.has_key(key) or self._fileStore.has_key(key)
def keys(self):
! return self._memoryStore.keys() + self._fileStore.keys()
def clear(self):
! self._memoryStore.clear()
! self._fileStore.clear()
def MovetoMemory(self, key):
! global debug
! if debug: print ">> Moving %s to Memory" % key
! self._memoryStore[key] = self._fileStore[key]
! del self._fileStore[key]
def MovetoFile(self, key):
! global debug
! if debug: print ">> Moving %s to File" % key
! self._fileStore[key] = self._memoryStore[key]
! del self._memoryStore[key]
!
## Application support ##
--- 25,153 ----
def __init__(self, app):
+ # Create both a file store and a memory store
SessionStore.__init__(self, app)
self._fileStore = SessionFileStore.SessionFileStore(app)
self._memoryStore = SessionMemoryStore.SessionMemoryStore(app, restoreFiles=0)
self._memoryStore.clear() #fileStore will have the files on disk
! # moveToFileInterval specifies after what period of time a session is automatically moved to file
! self._moveToFileInterval = self.application().setting('DynamicSessionTimeout', 15) * 60
! # maxDynamicMemorySessions is what the user actually sets in Application.config
self._maxDynamicMemorySessions = self.application().setting('MaxDynamicMemorySessions', 10000)
+ # Used to keep track of sweeping the file store
self._fileSweepCount = 0
+ # Create a re-entrant lock for thread synchronization. The lock is used to protect
+ # all code that modifies the contents of the file store and all code that moves sessions
+ # between the file and memory stores, and is also used to protect code that searches in
+ # the file store for a session. Using the lock in this way avoids a bug that used to
+ # be in this code, where a session was temporarily neither in the file store nor in
+ # the memory store while it was being moved from file to memory.
+ self._lock = threading.RLock()
+
if debug:
print "SessionDynamicStore Initialized"
!
## Access ##
def __len__(self):
! self._lock.acquire()
! try:
! return len(self._memoryStore) + len(self._fileStore)
! finally:
! self._lock.release()
def __getitem__(self, key):
! # First try to grab the session from the memory store without locking,
! # for efficiency. Only if that fails do we acquire the lock and
! # look in the file store.
! try:
! return self._memoryStore[key]
! except KeyError:
! self._lock.acquire()
! try:
! if self._fileStore.has_key(key):
! self.MovetoMemory(key)
! #let it raise a KeyError otherwise
! return self._memoryStore[key]
! finally:
! self._lock.release()
+
def __setitem__(self, key, item):
self._memoryStore[key] = item
! # @@ 2001-12-11 gat: Seems like a waste of time to attempt to delete the
! # session from the file store on every single write operation. I see no
! # harm in commenting out the rest of this method.
! ## try:
! ## del self._fileStore[key]
! ## except KeyError:
! ## pass
def __delitem__(self, key):
! self._lock.acquire()
! try:
! try:
! del self._memoryStore[key]
! except KeyError:
! pass
! try:
! del self._fileStore[key]
! except KeyError:
! pass
! finally:
! self._lock.release()
def has_key(self, key):
! # First try to find the session in the memory store without locking,
! # for efficiency. Only if that fails do we acquire the lock and
! # look in the file store.
! if self._memoryStore.has_key(key):
! return 1
! self._lock.acquire()
! try:
! return self._memoryStore.has_key(key) or self._fileStore.has_key(key)
! finally:
! self._lock.release()
def keys(self):
! self._lock.acquire()
! try:
! return self._memoryStore.keys() + self._fileStore.keys()
! finally:
! self._lock.release()
def clear(self):
! self._lock.acquire()
! try:
! self._memoryStore.clear()
! self._fileStore.clear()
! finally:
! self._lock.release()
def MovetoMemory(self, key):
! self._lock.acquire()
! try:
! global debug
! if debug: print ">> Moving %s to Memory" % key
! self._memoryStore[key] = self._fileStore[key]
! del self._fileStore[key]
! finally:
! self._lock.release()
def MovetoFile(self, key):
! self._lock.acquire()
! try:
! global debug
! if debug: print ">> Moving %s to File" % key
! self._fileStore[key] = self._memoryStore[key]
! del self._memoryStore[key]
! finally:
! self._lock.release()
+
## Application support ##
***************
*** 107,112 ****
def storeAllSessions(self):
! for i in self._memoryStore.keys():
! self.MovetoFile(i)
def cleanStaleSessions(self, task=None):
--- 156,165 ----
def storeAllSessions(self):
! self._lock.acquire()
! try:
! for i in self._memoryStore.keys():
! self.MovetoFile(i)
! finally:
! self._lock.release()
def cleanStaleSessions(self, task=None):
***************
*** 118,122 ****
Ideally, intervalSweep would be run more often than the cleanStaleSessions functions
for the actual stores. This may need to wait until we get the TaskKit in place, though.
!
The problem is the FileStore.cleanStaleSessions can take a while to run.
So here, we only run the file sweep every fourth time.
--- 171,175 ----
Ideally, intervalSweep would be run more often than the cleanStaleSessions functions
for the actual stores. This may need to wait until we get the TaskKit in place, though.
!
The problem is the FileStore.cleanStaleSessions can take a while to run.
So here, we only run the file sweep every fourth time.
***************
*** 136,141 ****
self.intervalSweep()
!
! #It's OK for a session to moved from memory to file or vice versa in between the time we get the keys and the time we actually ask for the session's access time. It may take a while for the fileStore sweep to get completed.
--- 189,194 ----
self.intervalSweep()
!
! #It's OK for a session to moved from memory to file or vice versa in between the time we get the keys and the time we actually ask for the session's access time. It may take a while for the fileStore sweep to get completed.
***************
*** 144,148 ****
The interval function moves sessions from memory to file. and can be run more often than the full
cleanStaleSessions function.
!
"""
global debug
--- 197,201 ----
The interval function moves sessions from memory to file. and can be run more often than the full
cleanStaleSessions function.
!
"""
global debug
***************
*** 151,217 ****
print "Memory Sessions: %s FileSessions: %s" % (len(self._memoryStore), len(self._fileStore))
print "maxDynamicMemorySessions = %s" % self._maxDynamicMemorySessions
! print "moveToFileInterval = %s" % self.moveToFileInterval
!
now = time.time()
-
- delta = now - self.moveToFileInterval
- for i in self._memoryStore.keys():
- if self._memoryStore[i].lastAccessTime() < delta:
- self.MovetoFile(i)
if len(self._memoryStore) > self._maxDynamicMemorySessions:
! keys = self._memoryStore.keys()
! keys.sort(self.sortFunc)
excess = len(self._memoryStore) - self._maxDynamicMemorySessions
if debug:
print excess, "sessions beyond the limit"
! for i in keys[:-excess]:
! self.MovetoFile(i)
!
! ## This is disabled for now
! if self.intelligentSessionArchive:
! #If we didn't get enough, find a tighter interval for next time
! if len(self._memoryStore) > self._maxDynamicMemorySessions \
! and self.moveToFileInterval > self.minMemoryInterval:
! self.moveToFileInterval = self.findInterval()
!
! #IF we have excess capacity in memory, increase the interval
! elif len(self._memoryStore) < (.5 * self._maxDynamicMemorySessions)\
! and self.moveToFileInterval < self.maxMemoryInterval:
! self.moveToFileInterval = self.moveToFileInterval * 2
-
if debug: print "Finished interval Sweep at %s" % time.ctime(time.time())
if debug: print "Memory Sessions: %s FileSessions: %s" % (len(self._memoryStore), len(self._fileStore))
-
- def findInterval(self):
- """
- Intelligently (?) find a period of time that will get the memory store down to it's maximum level.
! Not used for now.
"""
! global debug
! if debug: print "Finding Interval"
!
! keys = self._memoryStore.keys()
! keys.sort(self.sortFunc)
! count = 0
! while count < self._maxDynamicMemorySessions:
! count = count+1
! newinterval = time.time() - self._memoryStore[keys[count]].lastAccessTime()
!
! if debug: print "Found new interval: %s" % newinterval
! if newinterval > self.minMemoryInterval:
! return newinterval #5 minutes is minimum
! else:
! return self.minMemoryInterval
!
! def sortFunc(self,x,y):
! if self._memoryStore[x].lastAccessTime() > self._memoryStore[y].lastAccessTime():
! return -1
! else:
! return 1
!
!
--- 204,248 ----
print "Memory Sessions: %s FileSessions: %s" % (len(self._memoryStore), len(self._fileStore))
print "maxDynamicMemorySessions = %s" % self._maxDynamicMemorySessions
! print "moveToFileInterval = %s" % self._moveToFileInterval
!
now = time.time()
+ delta = now - self._moveToFileInterval
+ for i in self._memoryStore.keys():
+ try:
+ if self._memoryStore[i].lastAccessTime() < delta:
+ self.MovetoFile(i)
+ except KeyError:
+ pass
if len(self._memoryStore) > self._maxDynamicMemorySessions:
! keys = self.memoryKeysInAccessTimeOrder()
excess = len(self._memoryStore) - self._maxDynamicMemorySessions
if debug:
print excess, "sessions beyond the limit"
! for i in keys[:excess]:
! try:
! self.MovetoFile(i)
! except KeyError:
! pass
if debug: print "Finished interval Sweep at %s" % time.ctime(time.time())
if debug: print "Memory Sessions: %s FileSessions: %s" % (len(self._memoryStore), len(self._fileStore))
! def memoryKeysInAccessTimeOrder(self):
"""
! Returns memory store's keys in ascending order of last access time.
! """
! # This sorting technique is faster than using a comparison function.
! accessTimeAndKeys = []
! for key in self._memoryStore.keys():
! try:
! accessTimeAndKeys.append((self._memoryStore[key].lastAccessTime(), key))
! except KeyError:
! pass
! accessTimeAndKeys.sort()
! keys = []
! for accessTime, key in accessTimeAndKeys:
! keys.append(key)
! return keys
|