Thread: [SQL-CVS] SQLObject/SQLObject Cache.py,NONE,1.1 DBConnection.py,1.6,1.7 SQLObject.py,1.8,1.9
SQLObject is a Python ORM.
Brought to you by:
ianbicking,
phd
From: <ian...@us...> - 2003-03-14 09:14:31
|
Update of /cvsroot/sqlobject/SQLObject/SQLObject In directory sc8-pr-cvs1:/tmp/cvs-serv22961 Modified Files: DBConnection.py SQLObject.py Added Files: Cache.py Log Message: Fancier caching -- weak references, so objects can expire from cache, as well as having things expire from the cache. New Cache module that helps implement this. --- NEW FILE: Cache.py --- import threading from weakref import ref from time import time as now class CacheFactory(object): """ CacheFactory caches object creation. Each object should be referenced by a single hashable ID (note tuples of hashable values are also hashable). """ def __init__(self, expireFrequency=100, expireFraction=2, cache=True): """ Every expireFrequency times that an item is retrieved from this cache, the expire method is called. The expire method then expires an arbitrary fraction of the cached objects. The idea is at no time will the cache be entirely emptied, placing a potentially high load at that moment, but everything object will have its time to go eventually. The fraction is given as an integer, and one in that many objects are expired (i.e., the default is 1/2 of objects are expired). By setting cache to False, items won't be cached. However, in all cases a weak reference is kept to created objects, and if the object hasn't been garbage collected it will be returned. """ self.expireFrequency = expireFrequency self.expireCount = expireFrequency self.expireOffset = 0 self.expireFraction = expireFraction self.doCache = cache if self.doCache: self.cache = {} self.expiredCache = {} self.lock = threading.Lock() def get(self, id): if self.doCache: if self.expireCount > self.expireFrequency: # Two threads could hit the expire in a row, but # that's not so bad. At least by setting expireCount # back to zero right away we avoid this. The expire # method has a lock, so it's threadsafe. self.expireCount = 0 self.expire() try: return self.cache[id] except KeyError: pass self.lock.acquire() try: val = self.cache[id] except KeyError: pass else: self.lock.release() return val try: val = self.expiredCache[id]() except KeyError: return None else: del self.expiredCache[id] if val is None: return None self.cache[id] = val self.lock.release() return val else: try: val = self.expiredCache[id]() if val is not None: return val except KeyError: pass self.lock.acquire() try: val = self.expiredCache[id]() except KeyError: return None else: if val is None: del self.expiredCache[id] return None self.lock.release() return val def put(self, id, obj): if self.doCache: self.cache[id] = obj else: self.expiredCache[id] = ref(obj) self.lock.release() def created(self, id, obj): if self.doCache: self.cache[id] = obj else: self.expiredCache[id] = ref(obj) def expire(self): self.lock.acquire() keys = self.cache.keys() for i in xrange(self.expireOffset, len(keys), self.expireFraction): id = keys[i] self.expiredCache[id] = ref(self.cache[id]) del self.cache[id] # This offset tries to balance out which objects we expire, so # no object will just hang out in the cache forever. self.expireOffset = (self.expiredOffset + 1) % self.expireFraction self.lock.release() class CacheSet(object): def __init__(self, *args, **kw): self.caches = {} self.args = args self.kw = kw def get(self, id, cls): try: return self.caches[cls.__name__].get(id) except KeyError: self.caches[cls.__name__] = CacheFactory(*self.args, **self.kw) return self.caches[cls.__name__].get(id) def put(self, id, cls, obj): self.caches[cls.__name__].put(id, obj) def created(self, id, cls, obj): try: self.caches[cls.__name__].created(id, obj) except KeyError: self.caches[cls.__name__] = CacheFactory(*self.args, **self.kw) self.caches[cls.__name__].created(id, obj) Index: DBConnection.py =================================================================== RCS file: /cvsroot/sqlobject/SQLObject/SQLObject/DBConnection.py,v retrieving revision 1.6 retrieving revision 1.7 diff -C2 -d -r1.6 -r1.7 *** DBConnection.py 14 Mar 2003 03:52:01 -0000 1.6 --- DBConnection.py 14 Mar 2003 08:59:24 -0000 1.7 *************** *** 3,6 **** --- 3,7 ---- import threading import SQLBuilder + from Cache import CacheSet try: *************** *** 26,30 **** class DBConnection: ! def __init__(self, name=None, debug=False): if name: assert not _connections.has_key(name), 'A database by the name %s has already been created: %s' % (name, _connections[name]) --- 27,31 ---- class DBConnection: ! def __init__(self, name=None, debug=False, cache=True): if name: assert not _connections.has_key(name), 'A database by the name %s has already been created: %s' % (name, _connections[name]) *************** *** 32,37 **** self.name = name self.debug = debug ! self.instanceCache = {} ! self.instanceCacheLock = threading.Lock() --- 33,37 ---- self.name = name self.debug = debug ! self.cache = CacheSet(cache=cache) Index: SQLObject.py =================================================================== RCS file: /cvsroot/sqlobject/SQLObject/SQLObject/SQLObject.py,v retrieving revision 1.8 retrieving revision 1.9 diff -C2 -d -r1.8 -r1.9 *** SQLObject.py 14 Mar 2003 03:52:01 -0000 1.8 --- SQLObject.py 14 Mar 2003 08:59:24 -0000 1.9 *************** *** 409,447 **** id = int(id) - # If there is no specific connection given (so we don't - # care about transactions), we cache instances for the - # entire class, and we use the class cache: if connection is None: ! cache = cls._SO_instanceCache ! lock = cls._SO_instanceCacheLock ! # Otherwise we get the cache that belongs to the ! # connection: else: ! cache = connection.instanceCache.setdefault(cls.__name__, {}) ! lock = connection.instanceCacheLock ! # We're optimistic that we find a cached instance, if ! # so we can return it right away: ! if cache.has_key(id): ! return cache[id] ! else: ! # But if now we need to make a lock to make sure ! # no other threads make an instance while we're ! # making one: ! lock.acquire() ! # And just in case one has already finished in ! # the short time it took us to get here: ! if cache.has_key(id): ! lock.release() ! return cache[id] ! # Create the object, put it in the cache: ! inst = object.__new__(cls, id, connection) ! cache[id] = inst ! # Use the _init function -- __init__ can't be ! # used, since it's called even when we return an ! # already-created instance from the cache: ! inst._init(id, connection) ! lock.release() ! return inst def _init(self, id, connection=None): --- 409,423 ---- id = int(id) if connection is None: ! cache = cls._connection.cache else: ! cache = connection.cache ! val = cache.get(id, cls) ! if val is None: ! val = object.__new__(cls, id, connection) ! val._init(id, connection) ! cache.put(id, cls, val) ! return val def _init(self, id, connection=None): *************** *** 635,655 **** del self._SO_createValues del self._SO_creating ! if self._SO_perConnection: ! cache = self._connection.instanceCache.setdefault(self.__class__.__name__, {}) ! lock = self._connection.instanceCacheLock ! else: ! cache = self._SO_instanceCache ! lock = self._SO_instanceCacheLock ! # Everyone has to wait while we do the insert, so that a ! # select can't recreate this object during these next ! # moments. But at least we know that this object can't ! # be in the cache yet. ! lock.acquire() # Do the insert -- most of the SQL in this case is left # up to DBConnection, since getting a new ID is # non-standard. id = self._connection.queryInsertID(self._table, names, values) ! cache[id] = self ! lock.release() self._init(id) --- 611,621 ---- del self._SO_createValues del self._SO_creating ! # Do the insert -- most of the SQL in this case is left # up to DBConnection, since getting a new ID is # non-standard. id = self._connection.queryInsertID(self._table, names, values) ! cache = self._connection.cache ! cache.created(id, self.__class__, self) self._init(id) *************** *** 694,698 **** cls._connection.createTable(cls) if createJoinTables: ! self.createJoinTables(ifExists=ifExists) createTable = classmethod(createTable) --- 660,664 ---- cls._connection.createTable(cls) if createJoinTables: ! cls.createJoinTables(ifExists=ifExists) createTable = classmethod(createTable) *************** *** 701,705 **** # but right now it's making tables directly. result = [] ! for join in soClass._joins: if not join.hasIntermediateTable(): continue --- 667,671 ---- # but right now it's making tables directly. result = [] ! for join in cls._joins: if not join.hasIntermediateTable(): continue *************** *** 711,715 **** continue if ifExists and \ ! self._connection.tableExists(join.intermediateTable): continue self._connection.query( --- 677,681 ---- continue if ifExists and \ ! cls._connection.tableExists(join.intermediateTable): continue self._connection.query( *************** *** 720,723 **** --- 686,690 ---- join.otherColumn, self.joinSQLType(join))) + createJoinTables = classmethod(createJoinTables) def clearTable(cls): |