Thread: [Assorted-commits] SF.net SVN: assorted: [584] python-commons/tags
Brought to you by:
yangzhang
From: <yan...@us...> - 2008-03-03 04:55:28
|
Revision: 584 http://assorted.svn.sourceforge.net/assorted/?rev=584&view=rev Author: yangzhang Date: 2008-03-02 20:55:33 -0800 (Sun, 02 Mar 2008) Log Message: ----------- tagged 0.2 release Added Paths: ----------- python-commons/tags/0.2/ python-commons/tags/0.2/README python-commons/tags/0.2/publish.bash python-commons/tags/0.2/setup.py python-commons/tags/0.2/src/commons/__init__.py python-commons/tags/0.2/src/commons/files.py python-commons/tags/0.2/src/commons/setup.py Removed Paths: ------------- python-commons/tags/0.2/README python-commons/tags/0.2/setup.py python-commons/tags/0.2/src/commons/__init__.py python-commons/tags/0.2/src/commons/files.py python-commons/tags/0.2/src/commons/setup.py Copied: python-commons/tags/0.2 (from rev 473, python-commons/trunk) Deleted: python-commons/tags/0.2/README =================================================================== --- python-commons/trunk/README 2008-02-19 09:09:41 UTC (rev 473) +++ python-commons/tags/0.2/README 2008-03-03 04:55:33 UTC (rev 584) @@ -1,5 +0,0 @@ -To install, run setup.py or setup.bash. - -Future releases will come with scripts to build the epydoc API -documentation, but for now please refer to the project homepage for -the full documentation: http://assorted.sf.net/python-commons Copied: python-commons/tags/0.2/README (from rev 565, python-commons/trunk/README) =================================================================== --- python-commons/tags/0.2/README (rev 0) +++ python-commons/tags/0.2/README 2008-03-03 04:55:33 UTC (rev 584) @@ -0,0 +1,35 @@ +[documentation](doc) + +Overview +-------- + +Python Commons is a general-purpose library for Python. To get a sense of +what it provides, please glance over the [documentation](doc). + +Requirements +------------ + +- [Python](http://python.org/) 2.5 +- [setuptools](http://peak.telecommunity.com/DevCenter/setuptools) 0.6 + +Certain sub-modules have extra requirements: + +- `async` requires [Twisted](http://twistedmatrix.com/trac/) 2.5 +- `files` requires [path](http://www.jorendorff.com/articles/python/path/) 2.2 + +This library has only been tested on Linux. + +Setup +----- + +To install, run `easy_install python-commons`, or download the source tarball +and run `python setup.py install`. + +Related Work +------------ + +- [ASPN Cookbook]: a valuable repository of Python snippets +- [AIMA Utilities]: accompaniment to a popular AI textbook + +[ASPN Cookbook]: http://aspn.activestate.com/ASPN/Cookbook/Python +[AIMA Utilities]: http://aima.cs.berkeley.edu/python/utils.py Copied: python-commons/tags/0.2/publish.bash (from rev 566, python-commons/trunk/publish.bash) =================================================================== --- python-commons/tags/0.2/publish.bash (rev 0) +++ python-commons/tags/0.2/publish.bash 2008-03-03 04:55:33 UTC (rev 584) @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +post-stage() { + epydoc -o $stagedir/doc src/commons/ +} + +fullname='Python Commons' +version=0.2 +license=psf +websrcs=( README ) +rels=( pypi: ) +. assorted.bash "$@" Deleted: python-commons/tags/0.2/setup.py =================================================================== --- python-commons/trunk/setup.py 2008-02-19 09:09:41 UTC (rev 473) +++ python-commons/tags/0.2/setup.py 2008-03-03 04:55:33 UTC (rev 584) @@ -1,44 +0,0 @@ -#!/usr/bin/env python -# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- -# vim:ft=python:et:sw=4:ts=4 - -import os,sys -sys.path.insert( 0, os.path.join( os.path.dirname( sys.argv[0] ), 'src' ) ) -from commons import setup - -pkg_info_text = """ -Metadata-Version: 1.1 -Name: python-commons -Version: 0.2 -Author: Yang Zhang -Author-email: yaaang NOSPAM at REMOVECAPS gmail -Home-page: http://assorted.sourceforge.net/python-commons -Download-url: http://assorted.sourceforge.net/python-commons/download -Summary: Python Commons -License: Python Software Foundation License -Description: General-purpose library of utilities and extensions to the - standard library. -Keywords: Python,common,commons,utility,utilities,library,libraries -Platform: any -Provides: commons -Classifier: Development Status :: 4 - Beta -Classifier: Environment :: No Input/Output (Daemon) -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: Python Software Foundation License -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python -Classifier: Topic :: Communications -Classifier: Topic :: Database -Classifier: Topic :: Internet -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Classifier: Topic :: System -Classifier: Topic :: System :: Filesystems -Classifier: Topic :: System :: Logging -Classifier: Topic :: System :: Networking -Classifier: Topic :: Text Processing -Classifier: Topic :: Utilities -""" - -setup.run_setup( pkg_info_text, - #scripts = ['frontend/py_hotshot.py'], - ) Copied: python-commons/tags/0.2/setup.py (from rev 564, python-commons/trunk/setup.py) =================================================================== --- python-commons/tags/0.2/setup.py (rev 0) +++ python-commons/tags/0.2/setup.py 2008-03-03 04:55:33 UTC (rev 584) @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- +# vim:ft=python:et:sw=4:ts=4 + +import os,sys +sys.path.insert( 0, os.path.join( os.path.dirname( sys.argv[0] ), 'src' ) ) +from commons import setup + +pkg_info_text = """ +Metadata-Version: 1.1 +Name: python-commons +Version: 0.2 +Author: Yang Zhang +Author-email: yaaang NOSPAM at REMOVECAPS gmail +Home-page: http://assorted.sourceforge.net/python-commons +Summary: Python Commons +License: Python Software Foundation License +Description: General-purpose library of utilities and extensions to the + standard library. +Keywords: Python,common,commons,utility,utilities,library,libraries +Platform: any +Provides: commons +Classifier: Development Status :: 4 - Beta +Classifier: Environment :: No Input/Output (Daemon) +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Python Software Foundation License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Topic :: Communications +Classifier: Topic :: Database +Classifier: Topic :: Internet +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: System +Classifier: Topic :: System :: Filesystems +Classifier: Topic :: System :: Logging +Classifier: Topic :: System :: Networking +Classifier: Topic :: Text Processing +Classifier: Topic :: Utilities +""" + +setup.run_setup( pkg_info_text, + #scripts = ['frontend/py_hotshot.py'], + ) Deleted: python-commons/tags/0.2/src/commons/__init__.py =================================================================== --- python-commons/trunk/src/commons/__init__.py 2008-02-19 09:09:41 UTC (rev 473) +++ python-commons/tags/0.2/src/commons/__init__.py 2008-03-03 04:55:33 UTC (rev 584) @@ -1,48 +0,0 @@ -# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- -# vim:ft=python:et:sw=4:ts=4 - -""" -U{Python Commons<http://assorted.sf.net/python-commons>} is a collection of -general-purpose utilities released under the U{PSF -license<http://www.python.org/psf/license.html>} (the same license as what -Python is distributed under). - -You can download the latest releases -U{here<http://assorted.sf.net/python-commons/download>}. To install, -run setup.py or setup.bash. setup.bash requires -U{simple-setup<http://assorted.sf.net/shell-tools/simple-setup>} which -is part of U{shell-tools<http://assorted.sf.net/shell-tools>}. - -@author: Yang Zhang -@copyright: Yang Zhang unless otherwise noted -@license: PSF -""" - -__version__ = ( 0, 2, 0 ) -__all__ = [ 'async', - 'control', - 'decs', - 'environ', - 'exceps', - 'files', - 'interp', - 'log', - 'misc', - 'networking', - 'progress', - 'seqs', - 'servers', - 'startup', - 'strs', - 'structs', - 'threads', - 'trace' ] - -# TODO more resources: -# http://aima.cs.berkeley.edu/python/utils.py -# http://aspn.activestate.com/ASPN/Cookbook/Python -# http://aspn.activestate.com/ASPN/Cookbook/Python?&recipe_status=editors -# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/413486 -# interesting: -# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/259174 -# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/473790 Copied: python-commons/tags/0.2/src/commons/__init__.py (from rev 568, python-commons/trunk/src/commons/__init__.py) =================================================================== --- python-commons/tags/0.2/src/commons/__init__.py (rev 0) +++ python-commons/tags/0.2/src/commons/__init__.py 2008-03-03 04:55:33 UTC (rev 584) @@ -0,0 +1,40 @@ +# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- +# vim:ft=python:et:sw=4:ts=4 + +""" +U{Python Commons<http://assorted.sf.net/python-commons>} is a +general-purpose library. + +@author: Yang Zhang +@copyright: Yang Zhang unless otherwise noted +@license: PSF +""" + +__version__ = ( 0, 2, 0 ) +__all__ = [ 'async', + 'control', + 'decs', + 'environ', + 'exceps', + 'files', + 'interp', + 'log', + 'misc', + 'networking', + 'progress', + 'seqs', + 'servers', + 'startup', + 'strs', + 'structs', + 'threads', + 'trace' ] + +# TODO more resources: +# http://aima.cs.berkeley.edu/python/utils.py +# http://aspn.activestate.com/ASPN/Cookbook/Python +# http://aspn.activestate.com/ASPN/Cookbook/Python?&recipe_status=editors +# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/413486 +# interesting: +# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/259174 +# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/473790 Deleted: python-commons/tags/0.2/src/commons/files.py =================================================================== --- python-commons/trunk/src/commons/files.py 2008-02-19 09:09:41 UTC (rev 473) +++ python-commons/tags/0.2/src/commons/files.py 2008-03-03 04:55:33 UTC (rev 584) @@ -1,115 +0,0 @@ -# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- -# vim:ft=python:et:sw=4:ts=4 - -""" -File and directory manipulation. - -@var invalid_filename_chars: The characters which are usually -prohibited on most modern file systems. - -@var invalid_filename_chars_regex: A regex character class constructed -from L{invalid_filename_chars}. -""" - -from __future__ import with_statement - -import os, re, tempfile - -from path import path - -def soft_makedirs( path ): - """ - Emulate C{mkdir -p} (doesn't complain if it already exists). - - @param path: The path of the directory to create. - @type path: str - - @raise OSError: If it cannot create the directory. It only - swallows OS error 17. - """ - try: - os.makedirs( path ) - except OSError, ex: - if ex.errno == 17: - pass - else: - raise - -def temp_dir( base_dir_name, do_create_subdir = True ): - """ - Get a temporary directory without polluting top-level /tmp. This follows - Ubuntu's conventions, choosing a temporary directory name based on - the given name plus the user name to avoid user conflicts. - - @param base_dir_name: The "name" of the temporary directory. This - is usually identifies the purpose of the directory, or the - application to which the temporary directory belongs. E.g., if joe - calls passes in C{"ssh-agent"} on a standard Linux/Unix system, - then the full path of the temporary directory will be - C{"/tmp/ssh-agent-joe"}. - @type base_dir_name: str - - @param do_create_subdir: If C{True}, then creates a - sub-sub-directory within the temporary sub-directory (and returns - the path to that). The sub-sub-directory's name is randomized - (uses L{tempfile.mkdtemp}). - @type do_create_subdir: bool - - @return: The path to the temporary (sub-)sub-directory. - @rtype: str - """ - base_dir_name += '-' + os.environ[ 'USER' ] - base_dir = paths.path( tempfile.gettempdir() ) / base_dir_name - soft_makedirs( base_dir ) - if do_create_subdir: - return tempfile.mkdtemp( dir = base_dir ) - else: - return base_dir - -invalid_filename_chars = r'*|\/:<>?' -invalid_filename_chars_regex = r'[*|\\\/:<>?]' - -def cleanse_filename( filename ): - """ - Replaces all problematic characters in a filename with C{"_"}, as - specified by L{invalid_filename_chars}. - - @param filename: The filename to cleanse. - @type filename: str - """ - pattern = invalid_filename_chars_regex - return re.sub( pattern, '_', filename ) - -class disk_double_buffer( object ): - """ - A simple disk double-buffer. One file is for reading, the other is for - writing, and a facility for swapping the two roles is provided. - """ - def __init__( self, path_base, do_persist = True ): - self.paths = map( path, [ path_base + '.0', path_base + '.1' ] ) - self.do_persist = do_persist - self.switch_status = path( path_base + '.switched' ) - if not do_persist or not self.switch_status.exists(): - self.w, self.r = 0, 1 # default - else: - self.w, self.r = 1, 0 - self.reload_files() - def reload_files( self ): - self.writer = file( self.paths[ self.w ], 'w' ) - if not self.paths[ self.r ].exists(): - self.paths[ self.r ].touch() - self.reader = file( self.paths[ self.r ] ) - def switch( self ): - self.close() - if self.do_persist: - if self.w == 0: self.switch_status.touch() - else: self.switch_status.remove() - self.r, self.w = self.w, self.r - self.reload_files() - def write( self, x ): - self.writer.write( x ) - def read( self, len = 8192 ): - return self.reader.read( len ) - def close( self ): - self.reader.close() - self.writer.close() Copied: python-commons/tags/0.2/src/commons/files.py (from rev 567, python-commons/trunk/src/commons/files.py) =================================================================== --- python-commons/tags/0.2/src/commons/files.py (rev 0) +++ python-commons/tags/0.2/src/commons/files.py 2008-03-03 04:55:33 UTC (rev 584) @@ -0,0 +1,115 @@ +# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- +# vim:ft=python:et:sw=4:ts=4 + +""" +File and directory manipulation. + +@var invalid_filename_chars: The characters which are usually +prohibited on most modern file systems. + +@var invalid_filename_chars_regex: A regex character class constructed +from L{invalid_filename_chars}. +""" + +from __future__ import with_statement + +import os, re, tempfile + +from path import path + +def soft_makedirs( path ): + """ + Emulate C{mkdir -p} (doesn't complain if it already exists). + + @param path: The path of the directory to create. + @type path: str + + @raise OSError: If it cannot create the directory. It only + swallows OS error 17. + """ + try: + os.makedirs( path ) + except OSError, ex: + if ex.errno == 17: + pass + else: + raise + +def temp_dir( base_dir_name, do_create_subdir = True ): + """ + Get a temporary directory without polluting top-level /tmp. This follows + Ubuntu's conventions, choosing a temporary directory name based on + the given name plus the user name to avoid user conflicts. + + @param base_dir_name: The "name" of the temporary directory. This + is usually identifies the purpose of the directory, or the + application to which the temporary directory belongs. E.g., if joe + calls passes in C{"ssh-agent"} on a standard Linux/Unix system, + then the full path of the temporary directory will be + C{"/tmp/ssh-agent-joe"}. + @type base_dir_name: str + + @param do_create_subdir: If C{True}, then creates a + sub-sub-directory within the temporary sub-directory (and returns + the path to that). The sub-sub-directory's name is randomized + (uses L{tempfile.mkdtemp}). + @type do_create_subdir: bool + + @return: The path to the temporary (sub-)sub-directory. + @rtype: str + """ + base_dir_name += '-' + os.environ[ 'USER' ] + base_dir = path( tempfile.gettempdir() ) / base_dir_name + soft_makedirs( base_dir ) + if do_create_subdir: + return tempfile.mkdtemp( dir = base_dir ) + else: + return base_dir + +invalid_filename_chars = r'*|\/:<>?' +invalid_filename_chars_regex = r'[*|\\\/:<>?]' + +def cleanse_filename( filename ): + """ + Replaces all problematic characters in a filename with C{"_"}, as + specified by L{invalid_filename_chars}. + + @param filename: The filename to cleanse. + @type filename: str + """ + pattern = invalid_filename_chars_regex + return re.sub( pattern, '_', filename ) + +class disk_double_buffer( object ): + """ + A simple disk double-buffer. One file is for reading, the other is for + writing, and a facility for swapping the two roles is provided. + """ + def __init__( self, path_base, do_persist = True ): + self.paths = map( path, [ path_base + '.0', path_base + '.1' ] ) + self.do_persist = do_persist + self.switch_status = path( path_base + '.switched' ) + if not do_persist or not self.switch_status.exists(): + self.w, self.r = 0, 1 # default + else: + self.w, self.r = 1, 0 + self.reload_files() + def reload_files( self ): + self.writer = file( self.paths[ self.w ], 'w' ) + if not self.paths[ self.r ].exists(): + self.paths[ self.r ].touch() + self.reader = file( self.paths[ self.r ] ) + def switch( self ): + self.close() + if self.do_persist: + if self.w == 0: self.switch_status.touch() + else: self.switch_status.remove() + self.r, self.w = self.w, self.r + self.reload_files() + def write( self, x ): + self.writer.write( x ) + def read( self, len = 8192 ): + return self.reader.read( len ) + def close( self ): + self.reader.close() + self.writer.close() Deleted: python-commons/tags/0.2/src/commons/setup.py =================================================================== --- python-commons/trunk/src/commons/setup.py 2008-02-19 09:09:41 UTC (rev 473) +++ python-commons/tags/0.2/src/commons/setup.py 2008-03-03 04:55:33 UTC (rev 584) @@ -1,98 +0,0 @@ -#!/usr/bin/env python -# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- -# vim:ft=python:et:sw=4:ts=4 - -""" -Common code for setup.py files. -""" - -arg_keys = """ -name -version -author -author_email -description: Summary -download_url: Download-url -long_description: Description -keywords: Keywords -url: Home-page -license -classifiers: Classifier -platforms: Platform -""" - -import sys -if not hasattr(sys, "version_info") or sys.version_info < (2, 3): - from distutils.core import setup - _setup = setup - def setup(**kwargs): - for key in [ - # distutils >= Python 2.3 args - # XXX probably download_url came in earlier than 2.3 - "classifiers", "download_url", - # setuptools args - "install_requires", "zip_safe", "test_suite", - ]: - if kwargs.has_key(key): - del kwargs[key] - # Only want packages keyword if this is a package, - # only want py_modules keyword if this is a single-file module, - # so get rid of packages or py_modules keyword as appropriate. - if kwargs["packages"] is None: - del kwargs["packages"] - else: - del kwargs["py_modules"] - apply(_setup, (), kwargs) -else: - from setuptools import setup, find_packages - -def run_setup( pkg_info_text, *orig_args, **orig_kwargs ): - list_keys = set( [ 'Classifier' ] ) - pkg_info = {} - for line in pkg_info_text.split( '\n' ): - if line.strip() != '': - if line.startswith( ' '*8 ): - pkg_info[ key ] += line[ 7 : ] - else: - key, value = line.split( ': ', 1 ) - if key in list_keys: - try: - pkg_info[ key ].append( value ) - except: - pkg_info[ key ] = [ value ] - else: - pkg_info[ key ] = value - - args_nontranslations = set() - args_translations = {} - for line in arg_keys.split( '\n' ): - if line.strip() != '': - splitted = line.split( ': ', 1 ) - dest_name = splitted[ 0 ] - if len( splitted ) == 2: - source_name = splitted[ 1 ] - args_translations[ source_name ] = dest_name - else: - args_nontranslations.add( dest_name ) - - args = {} - for key, value in pkg_info.iteritems(): - dest_name = None - try: - dest_name = args_translations[ key ] - except KeyError: - key = key.lower().replace('-','_') - if key in args_nontranslations: - dest_name = key - if dest_name is not None: - args[ dest_name ] = value - - # this also allows user to override our args - args.update( orig_kwargs ) - - setup( *orig_args, - package_dir = {'':'src'}, - packages = find_packages('src'), - zip_safe = True, - **args, - ) Copied: python-commons/tags/0.2/src/commons/setup.py (from rev 567, python-commons/trunk/src/commons/setup.py) =================================================================== --- python-commons/tags/0.2/src/commons/setup.py (rev 0) +++ python-commons/tags/0.2/src/commons/setup.py 2008-03-03 04:55:33 UTC (rev 584) @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- +# vim:ft=python:et:sw=4:ts=4 + +""" +Common code for setup.py files. +""" + +arg_keys = """ +name +version +author +author_email +description: Summary +download_url: Download-url +long_description: Description +keywords: Keywords +url: Home-page +license +classifiers: Classifier +platforms: Platform +""" + +import sys +if not hasattr(sys, "version_info") or sys.version_info < (2, 3): + from distutils.core import setup + _setup = setup + def setup(**kwargs): + for key in [ + # distutils >= Python 2.3 args + # XXX probably download_url came in earlier than 2.3 + "classifiers", "download_url", + # setuptools args + "install_requires", "zip_safe", "test_suite", + ]: + if kwargs.has_key(key): + del kwargs[key] + # Only want packages keyword if this is a package, + # only want py_modules keyword if this is a single-file module, + # so get rid of packages or py_modules keyword as appropriate. + if kwargs["packages"] is None: + del kwargs["packages"] + else: + del kwargs["py_modules"] + apply(_setup, (), kwargs) +else: + from setuptools import setup, find_packages + +def run_setup( pkg_info_text, *orig_args, **orig_kwargs ): + list_keys = set( [ 'Classifier' ] ) + pkg_info = {} + for line in pkg_info_text.split( '\n' ): + if line.strip() != '': + if line.startswith( ' '*8 ): + pkg_info[ key ] += line[ 7 : ] + else: + key, value = line.split( ': ', 1 ) + if key in list_keys: + try: + pkg_info[ key ].append( value ) + except: + pkg_info[ key ] = [ value ] + else: + pkg_info[ key ] = value + + args_nontranslations = set() + args_translations = {} + for line in arg_keys.split( '\n' ): + if line.strip() != '': + splitted = line.split( ': ', 1 ) + dest_name = splitted[ 0 ] + if len( splitted ) == 2: + source_name = splitted[ 1 ] + args_translations[ source_name ] = dest_name + else: + args_nontranslations.add( dest_name ) + + args = {} + for key, value in pkg_info.iteritems(): + dest_name = None + try: + dest_name = args_translations[ key ] + except KeyError: + key = key.lower().replace('-','_') + if key in args_nontranslations: + dest_name = key + if dest_name is not None: + args[ dest_name ] = value + + # this also allows user to override our args + args.update( orig_kwargs ) + args.update( { + 'package_dir': {'':'src'}, + 'packages': find_packages('src'), + 'zip_safe': True, + } ) + + setup( *orig_args, **args ) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <yan...@us...> - 2008-05-08 08:29:17
|
Revision: 724 http://assorted.svn.sourceforge.net/assorted/?rev=724&view=rev Author: yangzhang Date: 2008-05-08 01:29:25 -0700 (Thu, 08 May 2008) Log Message: ----------- tagged release 0.4 Added Paths: ----------- python-commons/tags/0.4/ python-commons/tags/0.4/README python-commons/tags/0.4/publish.bash python-commons/tags/0.4/setup.py python-commons/tags/0.4/src/commons/decs.py python-commons/tags/0.4/src/commons/files.py python-commons/tags/0.4/src/commons/misc.py python-commons/tags/0.4/src/commons/networking.py python-commons/tags/0.4/src/commons/seqs.py python-commons/tags/0.4/src/commons/setup.py Removed Paths: ------------- python-commons/tags/0.4/README python-commons/tags/0.4/publish.bash python-commons/tags/0.4/setup.py python-commons/tags/0.4/src/commons/decs.py python-commons/tags/0.4/src/commons/files.py python-commons/tags/0.4/src/commons/misc.py python-commons/tags/0.4/src/commons/networking.py python-commons/tags/0.4/src/commons/seqs.py python-commons/tags/0.4/src/commons/setup.py Copied: python-commons/tags/0.4 (from rev 679, python-commons/trunk) Deleted: python-commons/tags/0.4/README =================================================================== --- python-commons/trunk/README 2008-04-24 16:13:04 UTC (rev 679) +++ python-commons/tags/0.4/README 2008-05-08 08:29:25 UTC (rev 724) @@ -1,35 +0,0 @@ -[documentation](doc) - -Overview --------- - -Python Commons is a general-purpose library for Python. To get a sense of -what it provides, please glance over the [documentation](doc). - -Requirements ------------- - -- [Python](http://python.org/) 2.5 -- [setuptools](http://peak.telecommunity.com/DevCenter/setuptools) 0.6 - -Certain sub-modules have extra requirements: - -- `async` requires [Twisted](http://twistedmatrix.com/trac/) 2.5 -- `files` requires [path](http://www.jorendorff.com/articles/python/path/) 2.2 - -This library has only been tested on Linux. - -Setup ------ - -To install, run `easy_install python-commons`, or download the source tarball -and run `python setup.py install`. - -Related Work ------------- - -- [ASPN Cookbook]: a valuable repository of Python snippets -- [AIMA Utilities]: accompaniment to a popular AI textbook - -[ASPN Cookbook]: http://aspn.activestate.com/ASPN/Cookbook/Python -[AIMA Utilities]: http://aima.cs.berkeley.edu/python/utils.py Copied: python-commons/tags/0.4/README (from rev 723, python-commons/trunk/README) =================================================================== --- python-commons/tags/0.4/README (rev 0) +++ python-commons/tags/0.4/README 2008-05-08 08:29:25 UTC (rev 724) @@ -0,0 +1,65 @@ +[documentation](doc) + +Overview +-------- + +Python Commons is a general-purpose library for Python. To get a sense of +what it provides, please glance over the [documentation](doc). + +Requirements +------------ + +- [Python](http://python.org/) 2.5 +- [setuptools](http://peak.telecommunity.com/DevCenter/setuptools) 0.6 + +Certain sub-modules have extra requirements: + +- `async` requires [Twisted](http://twistedmatrix.com/trac/) 2.5 +- `files` requires [path](http://www.jorendorff.com/articles/python/path/) 2.2 + +This library has only been tested on Linux. + +Setup +----- + +To install, run `easy_install python-commons`, or download the source tarball +and run `python setup.py install`. + +Related Work +------------ + +- [ASPN Cookbook]: a valuable repository of Python snippets +- [AIMA Utilities]: accompaniment to a popular AI textbook + +[ASPN Cookbook]: http://aspn.activestate.com/ASPN/Cookbook/Python +[AIMA Utilities]: http://aima.cs.berkeley.edu/python/utils.py + +Changes +------- + +version 0.4 + +- removed extraneous debug print statements +- added `logout()` context manager +- added `seq()`, `default_if_none()` +- fixed missing `import` bug +- released for [Mailing List + Filter](http://assorted.sf.net/mailing-list-filter/) + +version 0.3 + +- added versioned guards +- added file memoization +- added retry with exp backoff +- added `countstep()` +- released for + [gbookmark2delicious](http://gbookmark2delicious.googlecode.com/) + +version 0.2 + +- added `clients`, `setup` +- released for [icedb](http://cartel.csail.mit.edu/icedb/) + +version 0.1 + +- initial release Deleted: python-commons/tags/0.4/publish.bash =================================================================== --- python-commons/trunk/publish.bash 2008-04-24 16:13:04 UTC (rev 679) +++ python-commons/tags/0.4/publish.bash 2008-05-08 08:29:25 UTC (rev 724) @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -post-stage() { - epydoc -o $stagedir/doc src/commons/ -} - -fullname='Python Commons' -version=0.2 -license=psf -websrcs=( README ) -rels=( pypi: ) -. assorted.bash "$@" Copied: python-commons/tags/0.4/publish.bash (from rev 723, python-commons/trunk/publish.bash) =================================================================== --- python-commons/tags/0.4/publish.bash (rev 0) +++ python-commons/tags/0.4/publish.bash 2008-05-08 08:29:25 UTC (rev 724) @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +post-stage() { + epydoc -o $stagedir/doc src/commons/ +} + +fullname='Python Commons' +version=0.4 +license=psf +websrcs=( README ) +rels=( pypi: ) +. assorted.bash "$@" Deleted: python-commons/tags/0.4/setup.py =================================================================== --- python-commons/trunk/setup.py 2008-04-24 16:13:04 UTC (rev 679) +++ python-commons/tags/0.4/setup.py 2008-05-08 08:29:25 UTC (rev 724) @@ -1,43 +0,0 @@ -#!/usr/bin/env python -# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- -# vim:ft=python:et:sw=4:ts=4 - -import os,sys -sys.path.insert( 0, os.path.join( os.path.dirname( sys.argv[0] ), 'src' ) ) -from commons import setup - -pkg_info_text = """ -Metadata-Version: 1.1 -Name: python-commons -Version: 0.2 -Author: Yang Zhang -Author-email: yaaang NOSPAM at REMOVECAPS gmail -Home-page: http://assorted.sourceforge.net/python-commons -Summary: Python Commons -License: Python Software Foundation License -Description: General-purpose library of utilities and extensions to the - standard library. -Keywords: Python,common,commons,utility,utilities,library,libraries -Platform: any -Provides: commons -Classifier: Development Status :: 4 - Beta -Classifier: Environment :: No Input/Output (Daemon) -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: Python Software Foundation License -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python -Classifier: Topic :: Communications -Classifier: Topic :: Database -Classifier: Topic :: Internet -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Classifier: Topic :: System -Classifier: Topic :: System :: Filesystems -Classifier: Topic :: System :: Logging -Classifier: Topic :: System :: Networking -Classifier: Topic :: Text Processing -Classifier: Topic :: Utilities -""" - -setup.run_setup( pkg_info_text, - #scripts = ['frontend/py_hotshot.py'], - ) Copied: python-commons/tags/0.4/setup.py (from rev 723, python-commons/trunk/setup.py) =================================================================== --- python-commons/tags/0.4/setup.py (rev 0) +++ python-commons/tags/0.4/setup.py 2008-05-08 08:29:25 UTC (rev 724) @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- +# vim:ft=python:et:sw=4:ts=4 + +import os,sys +sys.path.insert( 0, os.path.join( os.path.dirname( sys.argv[0] ), 'src' ) ) +from commons import setup + +pkg_info_text = """ +Metadata-Version: 1.1 +Name: python-commons +Version: 0.4 +Author: Yang Zhang +Author-email: yaaang NOSPAM at REMOVECAPS gmail +Home-page: http://assorted.sourceforge.net/python-commons +Summary: Python Commons +License: Python Software Foundation License +Description: General-purpose library of utilities and extensions to the + standard library. +Keywords: Python,common,commons,utility,utilities,library,libraries +Platform: any +Provides: commons +Classifier: Development Status :: 4 - Beta +Classifier: Environment :: No Input/Output (Daemon) +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Python Software Foundation License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Topic :: Communications +Classifier: Topic :: Database +Classifier: Topic :: Internet +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: System +Classifier: Topic :: System :: Filesystems +Classifier: Topic :: System :: Logging +Classifier: Topic :: System :: Networking +Classifier: Topic :: Text Processing +Classifier: Topic :: Utilities +""" + +setup.run_setup( pkg_info_text, + #scripts = ['frontend/py_hotshot.py'], + ) Deleted: python-commons/tags/0.4/src/commons/decs.py =================================================================== --- python-commons/trunk/src/commons/decs.py 2008-04-24 16:13:04 UTC (rev 679) +++ python-commons/tags/0.4/src/commons/decs.py 2008-05-08 08:29:25 UTC (rev 724) @@ -1,92 +0,0 @@ -# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- -# vim:ft=python:et:sw=4:ts=4 - -""" -Decorators and decorator utilities. - -@todo: Move the actual decorators to modules based on their topic. -""" - -import functools, inspect, xmlrpclib - -def wrap_callable(any_callable, before, after): - """ - Wrap any callable with before/after calls. - - From the Python Cookbook. Modified to support C{None} for - C{before} or C{after}. - - @copyright: O'Reilly Media - - @param any_callable: The function to decorate. - @type any_callable: function - - @param before: The pre-processing procedure. If this is C{None}, then no pre-processing will be done. - @type before: function - - @param after: The post-processing procedure. If this is C{None}, then no post-processing will be done. - @type after: function - """ - def _wrapped(*a, **kw): - if before is not None: - before( ) - try: - return any_callable(*a, **kw) - finally: - if after is not None: - after( ) - # In 2.4, only: _wrapped.__name__ = any_callable.__name__ - return _wrapped - -class GenericWrapper( object ): - """ - Wrap all of an object's methods with before/after calls. This is - like a decorator for objects. - - From the I{Python Cookbook}. - - @copyright: O'Reilly Media - """ - def __init__(self, obj, before, after, ignore=( )): - # we must set into __dict__ directly to bypass __setattr__; so, - # we need to reproduce the name-mangling for double-underscores - clasname = 'GenericWrapper' - self.__dict__['_%s__methods' % clasname] = { } - self.__dict__['_%s__obj' % clasname] = obj - for name, method in inspect.getmembers(obj, inspect.ismethod): - if name not in ignore and method not in ignore: - self.__methods[name] = wrap_callable(method, before, after) - def __getattr__(self, name): - try: - return self.__methods[name] - except KeyError: - return getattr(self.__obj, name) - def __setattr__(self, name, value): - setattr(self.__obj, name, value) - -########################################################## - -def xmlrpc_safe(func): - """ - Makes a procedure "XMLRPC-safe" by returning 0 whenever the inner - function returns C{None}. This is useful because XMLRPC requires - return values, and 0 is commonly used when functions don't intend - to return anything. - - Also, if the procedure returns a boolean, it will be wrapped in - L{xmlrpclib.Boolean}. - - @param func: The procedure to decorate. - @type func: function - """ - @functools.wraps(func) - def wrapper(*args,**kwargs): - result = func(*args,**kwargs) - if result is not None: - if type( result ) == bool: - return xmlrpclib.Boolean( result ) - else: - return result - else: - return 0 - return wrapper Copied: python-commons/tags/0.4/src/commons/decs.py (from rev 687, python-commons/trunk/src/commons/decs.py) =================================================================== --- python-commons/tags/0.4/src/commons/decs.py (rev 0) +++ python-commons/tags/0.4/src/commons/decs.py 2008-05-08 08:29:25 UTC (rev 724) @@ -0,0 +1,156 @@ +# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- +# vim:ft=python:et:sw=4:ts=4 + +""" +Decorators and decorator utilities. + +@todo: Move the actual decorators to modules based on their topic. +""" + +from __future__ import with_statement +import functools, inspect, xmlrpclib +from cPickle import * + +def wrap_callable(any_callable, before, after): + """ + Wrap any callable with before/after calls. + + From the Python Cookbook. Modified to support C{None} for + C{before} or C{after}. + + @copyright: O'Reilly Media + + @param any_callable: The function to decorate. + @type any_callable: function + + @param before: The pre-processing procedure. If this is C{None}, then no pre-processing will be done. + @type before: function + + @param after: The post-processing procedure. If this is C{None}, then no post-processing will be done. + @type after: function + """ + def _wrapped(*a, **kw): + if before is not None: + before( ) + try: + return any_callable(*a, **kw) + finally: + if after is not None: + after( ) + # In 2.4, only: _wrapped.__name__ = any_callable.__name__ + return _wrapped + +class GenericWrapper( object ): + """ + Wrap all of an object's methods with before/after calls. This is + like a decorator for objects. + + From the I{Python Cookbook}. + + @copyright: O'Reilly Media + """ + def __init__(self, obj, before, after, ignore=( )): + # we must set into __dict__ directly to bypass __setattr__; so, + # we need to reproduce the name-mangling for double-underscores + clasname = 'GenericWrapper' + self.__dict__['_%s__methods' % clasname] = { } + self.__dict__['_%s__obj' % clasname] = obj + for name, method in inspect.getmembers(obj, inspect.ismethod): + if name not in ignore and method not in ignore: + self.__methods[name] = wrap_callable(method, before, after) + def __getattr__(self, name): + try: + return self.__methods[name] + except KeyError: + return getattr(self.__obj, name) + def __setattr__(self, name, value): + setattr(self.__obj, name, value) + +########################################################## + +def xmlrpc_safe(func): + """ + Makes a procedure "XMLRPC-safe" by returning 0 whenever the inner + function returns C{None}. This is useful because XMLRPC requires + return values, and 0 is commonly used when functions don't intend + to return anything. + + Also, if the procedure returns a boolean, it will be wrapped in + L{xmlrpclib.Boolean}. + + @param func: The procedure to decorate. + @type func: function + """ + @functools.wraps(func) + def wrapper(*args,**kwargs): + result = func(*args,**kwargs) + if result is not None: + if type( result ) == bool: + return xmlrpclib.Boolean( result ) + else: + return result + else: + return 0 + return wrapper + +########################################################## + +def file_memoized(serializer, deserializer, pathfunc): + """ + The string result of the given function is saved to the given path. + + Example:: + + @file_memoized(lambda x,f: f.write(x), + lambda f: f.read(), + lambda: "/tmp/cache") + def foo(): return "hello" + + @file_memoized(pickle.dump, + pickle.load, + lambda x,y: "/tmp/cache-%d-%d" % (x,y)) + def foo(x,y): return "hello %d %d" % (x,y) + + @param serializer: The function to serialize the return value into a + string. This should take the return value object and + the file object. + @type serializer: function + + @param deserializer: The function te deserialize the cache file contents + into the return value. This should take the file + object and return a string. + type: deserializer: function + + @param pathfunc: Returns the path where the files should be saved. This + should be able to take the same arguments as the original + function. + @type pathfunc: str + """ + def dec(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + p = pathfunc(*args, **kwargs) + try: + with file(p) as f: + return deserializer(f) + except IOError, (errno, errstr): + if errno != 2: raise + with file(p, 'w') as f: + x = func(*args, **kwargs) + serializer(x, f) + return x + return wrapper + return dec + +def file_string_memoized(pathfunc): + """ + Wrapper around L{file_memoized} that expects the decorated function to + return strings, so the string is written verbatim. + """ + return file_memoized(lambda x,f: f.write(x), lambda f: f.read(), pathfunc) + +def pickle_memoized(pathfunc): + """ + Wrapper around L{file_memoized} that uses pickle. + """ + return file_memoized(dump, load, pathfunc) Deleted: python-commons/tags/0.4/src/commons/files.py =================================================================== --- python-commons/trunk/src/commons/files.py 2008-04-24 16:13:04 UTC (rev 679) +++ python-commons/tags/0.4/src/commons/files.py 2008-05-08 08:29:25 UTC (rev 724) @@ -1,115 +0,0 @@ -# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- -# vim:ft=python:et:sw=4:ts=4 - -""" -File and directory manipulation. - -@var invalid_filename_chars: The characters which are usually -prohibited on most modern file systems. - -@var invalid_filename_chars_regex: A regex character class constructed -from L{invalid_filename_chars}. -""" - -from __future__ import with_statement - -import os, re, tempfile - -from path import path - -def soft_makedirs( path ): - """ - Emulate C{mkdir -p} (doesn't complain if it already exists). - - @param path: The path of the directory to create. - @type path: str - - @raise OSError: If it cannot create the directory. It only - swallows OS error 17. - """ - try: - os.makedirs( path ) - except OSError, ex: - if ex.errno == 17: - pass - else: - raise - -def temp_dir( base_dir_name, do_create_subdir = True ): - """ - Get a temporary directory without polluting top-level /tmp. This follows - Ubuntu's conventions, choosing a temporary directory name based on - the given name plus the user name to avoid user conflicts. - - @param base_dir_name: The "name" of the temporary directory. This - is usually identifies the purpose of the directory, or the - application to which the temporary directory belongs. E.g., if joe - calls passes in C{"ssh-agent"} on a standard Linux/Unix system, - then the full path of the temporary directory will be - C{"/tmp/ssh-agent-joe"}. - @type base_dir_name: str - - @param do_create_subdir: If C{True}, then creates a - sub-sub-directory within the temporary sub-directory (and returns - the path to that). The sub-sub-directory's name is randomized - (uses L{tempfile.mkdtemp}). - @type do_create_subdir: bool - - @return: The path to the temporary (sub-)sub-directory. - @rtype: str - """ - base_dir_name += '-' + os.environ[ 'USER' ] - base_dir = path( tempfile.gettempdir() ) / base_dir_name - soft_makedirs( base_dir ) - if do_create_subdir: - return tempfile.mkdtemp( dir = base_dir ) - else: - return base_dir - -invalid_filename_chars = r'*|\/:<>?' -invalid_filename_chars_regex = r'[*|\\\/:<>?]' - -def cleanse_filename( filename ): - """ - Replaces all problematic characters in a filename with C{"_"}, as - specified by L{invalid_filename_chars}. - - @param filename: The filename to cleanse. - @type filename: str - """ - pattern = invalid_filename_chars_regex - return re.sub( pattern, '_', filename ) - -class disk_double_buffer( object ): - """ - A simple disk double-buffer. One file is for reading, the other is for - writing, and a facility for swapping the two roles is provided. - """ - def __init__( self, path_base, do_persist = True ): - self.paths = map( path, [ path_base + '.0', path_base + '.1' ] ) - self.do_persist = do_persist - self.switch_status = path( path_base + '.switched' ) - if not do_persist or not self.switch_status.exists(): - self.w, self.r = 0, 1 # default - else: - self.w, self.r = 1, 0 - self.reload_files() - def reload_files( self ): - self.writer = file( self.paths[ self.w ], 'w' ) - if not self.paths[ self.r ].exists(): - self.paths[ self.r ].touch() - self.reader = file( self.paths[ self.r ] ) - def switch( self ): - self.close() - if self.do_persist: - if self.w == 0: self.switch_status.touch() - else: self.switch_status.remove() - self.r, self.w = self.w, self.r - self.reload_files() - def write( self, x ): - self.writer.write( x ) - def read( self, len = 8192 ): - return self.reader.read( len ) - def close( self ): - self.reader.close() - self.writer.close() Copied: python-commons/tags/0.4/src/commons/files.py (from rev 693, python-commons/trunk/src/commons/files.py) =================================================================== --- python-commons/tags/0.4/src/commons/files.py (rev 0) +++ python-commons/tags/0.4/src/commons/files.py 2008-05-08 08:29:25 UTC (rev 724) @@ -0,0 +1,171 @@ +# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- +# vim:ft=python:et:sw=4:ts=4 + +""" +File and directory manipulation. + +@var invalid_filename_chars: The characters which are usually +prohibited on most modern file systems. + +@var invalid_filename_chars_regex: A regex character class constructed +from L{invalid_filename_chars}. +""" + +from __future__ import with_statement + +import os, re, tempfile +from cPickle import * +from path import path + +def soft_makedirs( path ): + """ + Emulate C{mkdir -p} (doesn't complain if it already exists). + + @param path: The path of the directory to create. + @type path: str + + @raise OSError: If it cannot create the directory. It only + swallows OS error 17. + """ + try: + os.makedirs( path ) + except OSError, ex: + if ex.errno == 17: + pass + else: + raise + +def temp_dir( base_dir_name, do_create_subdir = True ): + """ + Get a temporary directory without polluting top-level /tmp. This follows + Ubuntu's conventions, choosing a temporary directory name based on + the given name plus the user name to avoid user conflicts. + + @param base_dir_name: The "name" of the temporary directory. This + is usually identifies the purpose of the directory, or the + application to which the temporary directory belongs. E.g., if joe + calls passes in C{"ssh-agent"} on a standard Linux/Unix system, + then the full path of the temporary directory will be + C{"/tmp/ssh-agent-joe"}. + @type base_dir_name: str + + @param do_create_subdir: If C{True}, then creates a + sub-sub-directory within the temporary sub-directory (and returns + the path to that). The sub-sub-directory's name is randomized + (uses L{tempfile.mkdtemp}). + @type do_create_subdir: bool + + @return: The path to the temporary (sub-)sub-directory. + @rtype: str + """ + base_dir_name += '-' + os.environ[ 'USER' ] + base_dir = path( tempfile.gettempdir() ) / base_dir_name + soft_makedirs( base_dir ) + if do_create_subdir: + return tempfile.mkdtemp( dir = base_dir ) + else: + return base_dir + +invalid_filename_chars = r'*|\/:<>?' +invalid_filename_chars_regex = r'[*|\\\/:<>?]' + +def cleanse_filename( filename ): + """ + Replaces all problematic characters in a filename with C{"_"}, as + specified by L{invalid_filename_chars}. + + @param filename: The filename to cleanse. + @type filename: str + """ + pattern = invalid_filename_chars_regex + return re.sub( pattern, '_', filename ) + +class disk_double_buffer( object ): + """ + A simple disk double-buffer. One file is for reading, the other is for + writing, and a facility for swapping the two roles is provided. + """ + def __init__( self, path_base, do_persist = True ): + self.paths = map( path, [ path_base + '.0', path_base + '.1' ] ) + self.do_persist = do_persist + self.switch_status = path( path_base + '.switched' ) + if not do_persist or not self.switch_status.exists(): + self.w, self.r = 0, 1 # default + else: + self.w, self.r = 1, 0 + self.reload_files() + def reload_files( self ): + self.writer = file( self.paths[ self.w ], 'w' ) + if not self.paths[ self.r ].exists(): + self.paths[ self.r ].touch() + self.reader = file( self.paths[ self.r ] ) + def switch( self ): + self.close() + if self.do_persist: + if self.w == 0: self.switch_status.touch() + else: self.switch_status.remove() + self.r, self.w = self.w, self.r + self.reload_files() + def write( self, x ): + self.writer.write( x ) + def read( self, len = 8192 ): + return self.reader.read( len ) + def close( self ): + self.reader.close() + self.writer.close() + +def versioned_guard(path, fresh_version): + """ + Maintain a version object. This is useful for working with versioned + caches. + + @param path: The path to the file containing the cached version object. + @type path: str + + @param fresh_version: The actual latest version that the cached version + should be compared against. + @type fresh_version: object (any type that can be compared) + + @return: True iff the cached version is obsolete (less than the fresh + version or doesn't exist). + @rtype: bool + """ + cache_version = None + try: + with file( path ) as f: cache_version = load(f) + except IOError, (errno, errstr): + if errno != 2: raise + if cache_version is None or fresh_version > cache_version: + with file( path, 'w' ) as f: dump(fresh_version, f) + return True + else: + return False + +def versioned_cache(version_path, fresh_version, cache_path, cache_func): + """ + If fresh_version is newer than the version in version_path, then invoke + cache_func and cache the result in cache_path (using pickle). + + Note the design flaw with L{versioned_guard}: the updated version value is + stored immediately, rather than after updating the cache. + + @param version_path: The path to the file version. + @type version_path: str + + @param fresh_version: The actual, up-to-date version value. + @type fresh_version: object (any type that can be compared) + + @param cache_path: The path to the cached data. + @type cache_path: str + + @param cache_func: The function that produces the fresh data to be cached. + @type cache_func: function (no arguments) + """ + if versioned_guard( version_path, fresh_version ): + # cache obsolete, force-fetch new data + result = cache_func() + with file(cache_path, 'w') as f: dump(result, f) + return result + else: + # cache up-to-date (should be available since dlcs-timestamp exists!) + with file(cache_path) as f: return load(f) Deleted: python-commons/tags/0.4/src/commons/misc.py =================================================================== --- python-commons/trunk/src/commons/misc.py 2008-04-24 16:13:04 UTC (rev 679) +++ python-commons/tags/0.4/src/commons/misc.py 2008-05-08 08:29:25 UTC (rev 724) @@ -1,48 +0,0 @@ -# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- -# vim:ft=python:et:sw=4:ts=4 - -from contextlib import * -from time import * - -""" -Miscellanea. -""" - -def generate_bit_fields(count): - """ - A generator of [2^i] for i from 0 to (count - 1). Useful for, - e.g., enumerating bitmask flags:: - - red, yellow, green, blue = generate_bit_fields(4) - color1 = blue - color2 = red | yellow - - @param count: The number of times to perform the left-shift. - @type count: int - """ - j = 1 - for i in xrange( count ): - yield j - j <<= 1 - -@contextmanager -def wall_clock(output): - """ - A simple timer for code sections. - - @param output: The resulting time is put into index 0 of L{output}. - @type output: index-writeable - - Example: - - t = [0] - with wall_clock(t): - sleep(1) - print "the sleep operation took %d seconds" % t[0] - """ - start = time() - try: - yield - finally: - end = time() - output[0] = end - start Copied: python-commons/tags/0.4/src/commons/misc.py (from rev 705, python-commons/trunk/src/commons/misc.py) =================================================================== --- python-commons/tags/0.4/src/commons/misc.py (rev 0) +++ python-commons/tags/0.4/src/commons/misc.py 2008-05-08 08:29:25 UTC (rev 724) @@ -0,0 +1,62 @@ +# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- +# vim:ft=python:et:sw=4:ts=4 + +from contextlib import * +from time import * + +""" +Miscellanea. +""" + +def generate_bit_fields(count): + """ + A generator of [2^i] for i from 0 to (count - 1). Useful for, + e.g., enumerating bitmask flags:: + + red, yellow, green, blue = generate_bit_fields(4) + color1 = blue + color2 = red | yellow + + @param count: The number of times to perform the left-shift. + @type count: int + """ + j = 1 + for i in xrange( count ): + yield j + j <<= 1 + +@contextmanager +def wall_clock(output): + """ + A simple timer for code sections. + + @param output: The resulting time is put into index 0 of L{output}. + @type output: index-writeable + + Example: + + t = [0] + with wall_clock(t): + sleep(1) + print "the sleep operation took %d seconds" % t[0] + """ + start = time() + try: + yield + finally: + end = time() + output[0] = end - start + +def default_if_none(x, d): + """ + Returns L{x} if it's not None, otherwise returns L{d}. + """ + if x is None: return d + else: return x + +def seq(f, g): + """ + Evaluate 0-ary functions L{f} then L{g}, returning L{g()}. + """ + f() + return g() Deleted: python-commons/tags/0.4/src/commons/networking.py =================================================================== --- python-commons/trunk/src/commons/networking.py 2008-04-24 16:13:04 UTC (rev 679) +++ python-commons/tags/0.4/src/commons/networking.py 2008-05-08 08:29:25 UTC (rev 724) @@ -1,39 +0,0 @@ -# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- -# vim:ft=python:et:sw=4:ts=4 - -""" -Networking tools. -""" - -import os, sys - -class NoMacAddrError( Exception ): pass - -def get_mac_addr(): - """ - Simply parses the output of C{ifconfig} or C{ipconfig} to estimate - this machine's IP address. This is not at all reliable, but tends - to work "well enough" for my own purposes. - - From U{http://mail.python.org/pipermail/python-list/2005-December/357300.html}. - - @copyright: Frank Millman - - Note that U{http://libdnet.sf.net/} provides this functionality and much - more. - """ - mac = None - if sys.platform == 'win32': - for line in os.popen("ipconfig /all"): - if line.lstrip().startswith('Physical Address'): - mac = line.split(':')[1].strip().replace('-',':') - break - else: - for line in os.popen("/sbin/ifconfig"): - if line.find('Ether') > -1: - mac = line.split()[4] - break - if mac is None: - raise NoMacAddrError - return mac - Copied: python-commons/tags/0.4/src/commons/networking.py (from rev 706, python-commons/trunk/src/commons/networking.py) =================================================================== --- python-commons/tags/0.4/src/commons/networking.py (rev 0) +++ python-commons/tags/0.4/src/commons/networking.py 2008-05-08 08:29:25 UTC (rev 724) @@ -0,0 +1,74 @@ +# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- +# vim:ft=python:et:sw=4:ts=4 + +""" +Networking tools. +""" + +import os, sys +from time import * +from contextlib import contextmanager + +class NoMacAddrError( Exception ): pass + +def get_mac_addr(): + """ + Simply parses the output of C{ifconfig} or C{ipconfig} to estimate + this machine's IP address. This is not at all reliable, but tends + to work "well enough" for my own purposes. + + From U{http://mail.python.org/pipermail/python-list/2005-December/357300.html}. + + @copyright: Frank Millman + + Note that U{http://libdnet.sf.net/} provides this functionality and much + more. + """ + mac = None + if sys.platform == 'win32': + for line in os.popen("ipconfig /all"): + if line.lstrip().startswith('Physical Address'): + mac = line.split(':')[1].strip().replace('-',':') + break + else: + for line in os.popen("/sbin/ifconfig"): + if line.find('Ether') > -1: + mac = line.split()[4] + break + if mac is None: + raise NoMacAddrError + return mac + +def retry_exp_backoff(initial_backoff, multiplier, func): + """ + Repeatedly invoke L{func} until it succeeds (returns non-None), with + exponentially growing backoff delay between each try. + + @param initial_backoff: The initial backoff. + @type initial_backoff: float + + @param multiplier: The amount by which the backoff is multiplied on each + failure. + @type multiplier: float + + @param func: The zero-argument function to be invoked that returns True on + success and False on failure. + @type func: function + + @return: The result of the function + """ + backoff = initial_backoff + while True: + res = func() + if res is not None: return res + print 'backing off for', backoff + sleep(backoff) + backoff = multiplier * backoff + +@contextmanager +def logout(x): + """ + A context manager for finally calling the C{logout()} method of an object. + """ + try: yield x + finally: x.logout() Deleted: python-commons/tags/0.4/src/commons/seqs.py =================================================================== --- python-commons/trunk/src/commons/seqs.py 2008-04-24 16:13:04 UTC (rev 679) +++ python-commons/tags/0.4/src/commons/seqs.py 2008-05-08 08:29:25 UTC (rev 724) @@ -1,357 +0,0 @@ -# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- -# vim:ft=python:et:sw=4:ts=4 - -from __future__ import ( absolute_import, with_statement ) - -from cStringIO import StringIO -from cPickle import * -from struct import pack, unpack -from contextlib import closing -from itertools import ( chain, count, ifilterfalse, islice, - izip, tee ) -from .log import warning - -""" -Sequences, streams, and generators. - -@var default_chunk_size: The default chunk size used by L{chunkify}. -""" - -default_chunk_size = 8192 - -def read_pickle( read, init = '', length_thresh = 100000 ): - """ - Given a reader function L{read}, reads in pickled objects from it. I am a - generator which yields unpickled objects. I assume that the pickling - is "safe," done using L{safe_pickle}. - - @param read: The reader function that reads from a stream. It should take - a single argument, the number of bytes to consume. - @type read: function - - @return: A tuple whose first element is the deserialized object or None if - EOF was encountered, and whose second element is the remainder bytes until - the EOF that were not consumed by unpickling. - @rtype: (object, str) - """ - with closing( StringIO() ) as sio: - obj = None # return this if we hit eof (not enough bytes read) - sio.write( init ) - - def read_until( target ): - remain = target - streamlen( sio ) - if remain > 0: - chunk = read( remain ) - # append to end - sio.seek(0,2) - sio.write( chunk ) - offset = streamlen( sio ) - sio.seek(0) - return offset >= target - - if read_until(4): - lengthstr = sio.read(4) - (length,) = unpack('i4', lengthstr) - if length_thresh is not None and length > length_thresh or \ - length <= 0: - warning( 'read_pickle', - 'got length', length, - 'streamlen', streamlen(sio), - 'first bytes %x %x %x %x' % tuple(map(ord,lengthstr)) ) - if read_until(length+4): - # start reading from right after header - sio.seek(4) - obj = load(sio) - - return ( obj, sio.read() ) - -def read_pickles( read ): - """ - Reads all the consecutively pickled objects from the L{read} function. - """ - while True: - pair = ( obj, rem ) = read_pickle( read ) - if obj is None: break - yield pair - -class safe_pickler( object ): - def __init__( self, protocol = HIGHEST_PROTOCOL ): - self.sio = StringIO() - self.pickler = Pickler( self.sio, protocol ) - def dumps( self, obj ): - """ - Pickle L{obj} but prepends the serialized length in bytes. - """ - self.pickler.clear_memo() - self.sio.seek(0) - self.pickler.dump(obj) - self.sio.truncate() - msg = self.sio.getvalue() - return pack('i4', self.sio.tell()) + msg - -def write_pickle( obj, write ): - """ - Write L{obj} using function L{write}, in a safe, pickle-able fashion. - """ - return write( safe_pickle( obj ) ) - -def streamlen( stream ): - """ - Get the length of a stream (e.g. file stream or StringIO). - Tries to restore the original position in the stream. - """ - orig_pos = stream.tell() - stream.seek(0,2) # seek to 0 relative to eof - length = stream.tell() # get the position - stream.seek(orig_pos) # return to orig_pos - return length - -def chunkify( stream, chunk_size = default_chunk_size ): - """ - Given an input stream (an object exposing a file-like interface), - reads data in from it one chunk at a time. This is a generator - which yields those chunks as they come. - - @param stream: The input stream. - @type stream: stream - - @param chunk_size: The size of the chunk (usually the number of - bytes to read). - @type chunk_size: int - """ - offset = 0 - while True: - chunk = stream.read( chunk_size ) - if not chunk: - break - yield offset, chunk - offset += len( chunk ) - -def total( iterable ): - """ - Counts the number of items in an iterable. Note that this will - consume the elements of the iterable, and if the iterable is - infinite, this will not halt. - - @param iterable: The iterable to count. - @type iterable - - @return: The number of elements consumed. - @rtype: int - """ - return sum( 1 for i in iterable ) - -#class FilePersistence(): -# def __init__( self ): -# -# -#class DbPersistence(): -# def __init__( self ): -# - -class ClosedError( Exception ): pass - -class PersistentConsumedSeq( object ): - """ - I generate C{[0, 1, ...]}, like L{count}, but I can also - save my state to disk. Similar to L{PersistentSeq}, but instead of - committing on each call to L{next}, require manual explicit calls - to L{commit}. I'm useful for generating unique IDs. - - Why not simply use L{PersistentSeq} instead of me? You usually - can. However, some applications use me for efficiency. For - instance, consider an application that generates a lot of network - packets (with sequence numbers), but only sends a small fraction - of them out onto the network. If we only want to guarantee the - uniqueness of sequence numbers that are exposed to the world, we - need only commit when upon sending a packet, and not on generating - a packet (L{next}). This could avoid excessive writes. - - @ivar seqno: The next sequence number to be generated. - @type seqno: int - """ - def __init__( self, path ): - """ - @param path: File to save my state in. I keep this file open. - @type path: str - """ - try: - self.log = file( path, 'r+' ) - except IOError, ex: - if ex.errno == 2: - self.log = file( path, 'w+' ) - else: - raise - contents = self.log.read() - if len( contents ) > 0: - self.seqno = int( contents ) - else: - self.seqno = 0 - self.max_commit = self.seqno - def next( self ): - """ - @return: The next number in the sequence. - @rtype: int - - @throw ClosedError: If I was previously L{close}d. - """ - if self.log is None: - raise ClosedError() - self.seqno += 1 - return self.seqno - 1 - def commit( self, seqno ): - """ - @param seqno: If this is the maximum committed sequence - number, then commit this sequence number (to disk). The - semantics will get weird if you pass in sequence numbers that - haven't been generated yet. - - @type seqno: int - - @return: The maximum sequence number ever committed (possibly - L{seqno}). - @rtype: int - - @throw ClosedError: If I was previously L{close}d. - """ - if self.log is None: - raise ClosedError() - if seqno > self.max_commit: - # TODO use a more flexible logging system that can switch - # between Python's logging module and Twisted's log module - self.max_commit = seqno - self.log.seek( 0 ) - # yes I write +1 here - self.log.write( str( seqno + 1 ) ) - self.log.truncate() - self.log.flush() - return self.max_commit - def close( self ): - """ - Closes the log file. No more operations can be performed. - """ - self.log.close() - self.log = None - -class PersistentSeq( PersistentConsumedSeq ): - """ - I generate C{[0, 1, ...]}, like L{count}, but I can also - save my state to disk. I save my state immediately to disk on each - call to L{next}. - """ - def __init__( self, path ): - """ - @param path: File to save my state in. I keep this file open. - @type path: str - """ - PersistentConsumedSeq.__init__( self, path ) - def next( self ): - """ - Generates the next number in the sequence and immediately - commits it. - """ - cur = PersistentConsumedSeq.next( self ) - self.commit( cur ) - return cur - -def pairwise(iterable): - "s -> (s0,s1), (s1,s2), (s2, s3), ..." - a, b = tee(iterable) - try: - b.next() - except StopIteration: - pass - return izip(a, b) - -def argmax(sequence, fn=None): - """Two usage patterns: - C{argmax([s0, s1, ...], fn)} - C{argmax([(fn(s0), s0), (fn(s1), s1), ...])} - Both return the si with greatest fn(si)""" - if fn is None: - return max(sequence)[1] - else: - return max((fn(e), e) for e in sequence)[1] - -def argmin(sequence, fn=None): - """Two usage patterns: - C{argmin([s0, s1, ...], fn)} - C{argmin([(fn(s0), s0), (fn(s1), s1), ...])} - Both return the si with smallest fn(si)""" - if fn is None: - return min(sequence)[1] - else: - return min((fn(e), e) for e in sequence)[1] - -def all(seq, pred=bool): - """ - Returns C{True} if C{pred(x) is True} for every element in the - iterable - """ - for elem in ifilterfalse(pred, seq): - return False - return True - -def concat(listOfLists): - return list(chain(*listOfLists)) - -def flatten( stream ): - """ - For each item yielded by L{gen}, if that item is itself an - iterator/generator, then I will recurse into C{flatten(gen)}; - otherwise, I'll yield the yielded item. Thus, I essentially - "flatten" out a tree of iterators. - - I test whether something is an iterator/generator simply by - checking to see if it has a C{next} attribute. Note that this - won't include any iterable, so things like L{list}s are yielded - like any regular item. This is my author's desired behavior! - - I am useful for coroutines, a la DeferredGenerators from Twisted. - - See also: - U{http://mail.python.org/pipermail/python-list/2003-October/232874.html} - """ - for item in stream: - if hasattr( item, 'next' ): - for item in flatten( item ): - yield item - else: - yield item - -def grouper(n, iterable, padvalue=None): - "grouper(3, 'abcdefg', 'x') --> ('a','b','c'), ('d','e','f'), ('g','x','x')" - return izip(*[chain(iterable, repeat(padvalue, n-1))]*n) - -def chunker( n, iterable, in_place = False ): - """ - Like L{grouper} but designed to scale for larger L{n}. Also, does - not perform padding. The end of the stream is reached when we - yield a chunk with fewer than L{n} items. - """ - i = -1 - chunk = [ None ] * n - for i, item in enumerate( iterable ): - chunk[ i % n ] = item - if ( i + 1 ) % n == 0: - yield chunk - if not in_place: chunk = [ None ] * n - else: - if i % n < n - 1: - del chunk[ ( i + 1 ) % n : ] - yield chunk - -def take(n, seq): - return list(islice(seq, n)) - -def delimit(sep, xs): - for x in xs: - yield x - break - for x in xs: - yield sep - yield x - -# TODO not quite right -def interleave(xs, ys): - return concat(izip( xs, ys )) Copied: python-commons/tags/0.4/src/commons/seqs.py (from rev 707, python-commons/trunk/src/commons/seqs.py) =================================================================== --- python-commons/tags/0.4/src/commons/seqs.py (rev 0) +++ python-commons/tags/0.4/src/commons/seqs.py 2008-05-08 08:29:25 UTC (rev 724) @@ -0,0 +1,366 @@ +# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- +# vim:ft=python:et:sw=4:ts=4 + +from __future__ import ( absolute_import, with_statement ) + +from cStringIO import StringIO +from cPickle import * +from struct import pack, unpack +from contextlib import closing +from itertools import ( chain, count, ifilterfalse, islice, + izip, repeat, tee ) +from .log import warning + +""" +Sequences, streams, and generators. + +@var default_chunk_size: The default chunk size used by L{chunkify}. +""" + +default_chunk_size = 8192 + +def read_pickle( read, init = '', length_thresh = 100000 ): + """ + Given a reader function L{read}, reads in pickled objects from it. I am a + generator which yields unpickled objects. I assume that the pickling + is "safe," done using L{safe_pickle}. + + @param read: The reader function that reads from a stream. It should take + a single argument, the number of bytes to consume. + @type read: function + + @return: A tuple whose first element is the deserialized object or None if + EOF was encountered, and whose second element is the remainder bytes until + the EOF that were not consumed by unpickling. + @rtype: (object, str) + """ + with closing( StringIO() ) as sio: + obj = None # return this if we hit eof (not enough bytes read) + sio.write( init ) + + def read_until( target ): + remain = target - streamlen( sio ) + if remain > 0: + chunk = read( remain ) + # append to end + sio.seek(0,2) + sio.write( chunk ) + offset = streamlen( sio ) + sio.seek(0) + return offset >= target + + if read_until(4): + lengthstr = sio.read(4) + (length,) = unpack('i4', lengthstr) + if length_thresh is not None and length > length_thresh or \ + length <= 0: + warning( 'read_pickle', + 'got length', length, + 'streamlen', streamlen(sio), + 'first bytes %x %x %x %x' % tuple(map(ord,lengthstr)) ) + if read_until(length+4): + # start reading from right after header + sio.seek(4) + obj = load(sio) + + return ( obj, sio.read() ) + +def read_pickles( read ): + """ + Reads all the consecutively pickled objects from the L{read} function. + """ + while True: + pair = ( obj, rem ) = read_pickle( read ) + if obj is None: break + yield pair + +class safe_pickler( object ): + def __init__( self, protocol = HIGHEST_PROTOCOL ): + self.sio = StringIO() + self.pickler = Pickler( self.sio, protocol ) + def dumps( self, obj ): + """ + Pickle L{obj} but prepends the serialized length in bytes. + """ + self.pickler.clear_memo() + self.sio.seek(0) + self.pickler.dump(obj) + self.sio.truncate() + msg = self.sio.getvalue() + return pack('i4', self.sio.tell()) + msg + +def write_pickle( obj, write ): + """ + Write L{obj} using function L{write}, in a safe, pickle-able fashion. + """ + return write( safe_pickle( obj ) ) + +def streamlen( stream ): + """ + Get the length of a stream (e.g. file stream or StringIO). + Tries to restore the original position in the stream. + """ + orig_pos = stream.tell() + stream.seek(0,2) # seek to 0 relative to eof + length = stream.tell() # get the position + stream.seek(orig_pos) # return to orig_pos + return length + +def chunkify( stream, chunk_size = default_chunk_size ): + """ + Given an input stream (an object exposing a file-like interface), + reads data in from it one chunk at a time. This is a generator + which yields those chunks as they come. + + @param stream: The input stream. + @type stream: stream + + @param chunk_size: The size of the chunk (usually the number of + bytes to read). + @type chunk_size: int + """ + offset = 0 + while True: + chunk = stream.read( chunk_size ) + if not chunk: + break + yield offset, chunk + offset += len( chunk ) + +def total( iterable ): + """ + Counts the number of items in an iterable. Note that this will + consume the elements of the iterable, and if the iterable is + infinite, this will not halt. + + @param iterable: The iterable to count. + @type iterable + + @return: The number of elements consumed. + @rtype: int + """ + return sum( 1 for i in iterable ) + +#class FilePersistence(): +# def __init__( self ): +# +# +#class DbPersistence(): +# def __init__( self ): +# + +class ClosedError( Exception ): pass + +class PersistentConsumedSeq( object ): + """ + I generate C{[0, 1, ...]}, like L{count}, but I can also + save my state to disk. Similar to L{PersistentSeq}, but instead of + committing on each call to L{next}, require manual explicit calls + to L{commit}. I'm useful for generating unique IDs. + + Why not simply use L{PersistentSeq} instead of me? You usually + can. However, some applications use me for efficiency. For + instance, consider an application that generates a lot of network + packets (with sequence numbers), but only sends a small fraction + of them out onto the network. If we only want to guarantee the + uniqueness of sequence numbers that are exposed to the world, we + need only commit when upon sending a packet, and not on generating + a packet (L{next}). This could avoid excessive writes. + + @ivar seqno: The next sequence number to be generated. + @type seqno: int + """ + def __init__( self, path ): + """ + @param path: File to save my state in. I keep this file open. + @type path: str + """ + try: + self.log = file( path, 'r+' ) + except IOError, ex: + if ex.errno == 2: + self.log = file( path, 'w+' ) + else: + raise + contents = self.log.read() + if len( contents ) > 0: + self.seqno = int( contents ) + else: + self.seqno = 0 + self.max_commit = self.seqno + def next( self ): + """ + @return: The next number in the sequence. + @rtype: int + + @throw ClosedError: If I was previously L{close}d. + """ + if self.log is None: + raise ClosedError() + self.seqno += 1 + return self.seqno - 1 + def commit( self, seqno ): + """ + @param seqno: If this is the maximum committed sequence + number, then commit this sequence number (to disk). The + semantics will get weird if you pass in sequence numbers that + haven't been generated yet. + + @type seqno: int + + @return: The maximum sequence number ever committed (possibly + L{seqno}). + @rtype: int + + @throw ClosedError: If I was previously L{close}d. + """ + if self.log is None: + raise ClosedError() + if seqno > self.max_commit: + # TODO use a more flexible logging system that can switch + # between Python's logging module and Twisted's log module + self.max_commit = seqno + self.log.seek( 0 ) + # yes I write +1 here + self.log.write( str( seqno + 1 ) ) + self.log.truncate() + self.log.flush() + return self.max_commit + def close( self ): + """ + Closes the log file. No more operations can be performed. + """ + self.log.close() + self.log = None + +class PersistentSeq( PersistentConsumedSeq ): + """ + I generate C{[0, 1, ...]}, like L{count}, but I can also + save my state to disk. I save my state immediately to disk on each + call to L{next}. + """ + def __init__( self, path ): + """ + @param path: File to save my state in. I keep this file open. + @type path: str + """ + PersistentConsumedSeq.__init__( self, path ) + def next( self ): + """ + Generates the next number in the sequence and immediately + commits it. + """ + cur = PersistentConsumedSeq.next( self ) + self.commit( cur ) + return cur + +def pairwise(iterable): + "s -> (s0,s1), (s1,s2), (s2, s3), ..." + a, b = tee(iterable) + try: + b.next() + except StopIteration: + pass + return izip(a, b) + +def argmax(sequence, fn=None): + """Two usage patterns: + C{argmax([s0, s1, ...], fn)} + C{argmax([(fn(s0), s0), (fn(s1), s1), ...])} + Both return the si with greatest fn(si)""" + if fn is None: + return max(sequence)[1] + else: + return max((fn(e), e) for e in sequence)[1] + +def argmin(sequence, fn=None): + """Two usage patterns: + C{argmin([s0, s1, ...], fn)} + C{argmin([(fn(s0), s0), (fn(s1), s1), ...])} + Both return the si with smallest fn(si)""" + if fn is None: + return min(sequence)[1] + else: + return min((fn(e), e) for e in sequence)[1] + +def all(seq, pred=bool): + """ + Returns C{True} if C{pred(x) is True} for every element in the + iterable + """ + for elem in ifilterfalse(pred, seq): + return False + return True + +def concat(listOfLists): + return list(chain(*listOfLists)) + +def flatten( stream ): + """ + For each item yielded by L{gen}, if that item is itself an + iterator/generator, then I will recurse into C{flatten(gen)}; + otherwise, I'll yield the yielded item. Thus, I essentially + "flatten" out a tree of iterators. + + I test whether something is an iterator/generator simply by + checking to see if it has a C{next} attribute. Note that this + won't include any iterable, so things like L{list}s are yielded + like any regular item. This is my author's desired behavior! + + I am useful for coroutines, a la DeferredGenerators from Twisted. + + See also: + U{http://mail.python.org/pipermail/python-list/2003-October/232874.html} + """ + for item in stream: + if hasattr( item, 'next' ): + for item in flatten( item ): + yield item + else: + yield item + +def grouper(n, iterable, padvalue=None): + "grouper(3, 'abcdefg', 'x') --> ('a','b','c'), ('d','e','f'), ('g','x','x')" + return izip(*[chain(iterable, repeat(padvalue, n-1))]*n) + +def chunker( n, iterable, in_place = False ): + """ + Like L{grouper} but designed to scale for larger L{n}. Also, does + not perform padding. The end of the stream is reached when we + yield a chunk with fewer than L{n} items. + """ + i = -1 + chunk = [ None ] * n + for i, item in enumerate( iterable ): + chunk[ i % n ] = item + if ( i + 1 ) % n == 0: + yield chunk + if not in_place: chunk = [ None ] * n + else: + if i % n < n - 1: + del chunk[ ( i + 1 ) % n : ] + yield chunk + +def countstep(start, step): + """ + Generate [start, start+step, start+2*step, start+3*step, ...]. + """ + i = start + while True: + yield i + i += step + +def take(n, seq): + return list(islice(seq, n)) + +def delimit(sep, xs): + for x in xs: + yield x + break + for x in xs: + yield sep + yield x + +# TODO not quite right +def interleave(xs, ys): + return concat(izip( xs, ys )) Deleted: python-commons/tags/0.4/src/commons/setup.py =================================================================== --- python-commons/trunk/src/commons/setup.py 2008-04-24 16:13:04 UTC (rev 679) +++ python-commons/tags/0.4/src/commons/setup.py 2008-05-08 08:29:25 UTC (rev 724) @@ -1,98 +0,0 @@ -#!/usr/bin/env python -# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- -# vim:ft=python:et:sw=4:ts=4 - -""" -Common code for setup.py files. -""" - -arg_keys = """ -name -version -author -author_email -description: Summary -download_url: Download-url -long_description: Description -keywords: Keywords -url: Home-page -license -classifiers: Classifier -platforms: Platform -""" - -import sys -if not hasattr(sys, "version_info") or sys.version_info < (2, 3): - from distutils.core import setup - _setup = setup - def setup(**kwargs): - for key in [ - # distutils >= Python 2.3 args - # XXX probably download_url came... [truncated message content] |
From: <yan...@us...> - 2008-05-14 18:02:58
|
Revision: 814 http://assorted.svn.sourceforge.net/assorted/?rev=814&view=rev Author: yangzhang Date: 2008-05-14 11:00:58 -0700 (Wed, 14 May 2008) Log Message: ----------- tagged 0.5 release! Modified Paths: -------------- python-commons/tags/0.5/publish.bash python-commons/tags/0.5/setup.py python-commons/tags/0.5/src/commons/__init__.py Added Paths: ----------- python-commons/tags/0.5/ python-commons/tags/0.5/README python-commons/tags/0.5/src/commons/setup.py python-commons/tags/0.5/src/commons/strs.py Removed Paths: ------------- python-commons/tags/0.5/README python-commons/tags/0.5/src/commons/setup.py python-commons/tags/0.5/src/commons/strs.py Copied: python-commons/tags/0.5 (from rev 791, python-commons/trunk) Deleted: python-commons/tags/0.5/README =================================================================== --- python-commons/trunk/README 2008-05-10 05:46:06 UTC (rev 791) +++ python-commons/tags/0.5/README 2008-05-14 18:00:58 UTC (rev 814) @@ -1,65 +0,0 @@ -[documentation](doc) - -Overview --------- - -Python Commons is a general-purpose library for Python. To get a sense of -what it provides, please glance over the [documentation](doc). - -Requirements ------------- - -- [Python](http://python.org/) 2.5 -- [setuptools](http://peak.telecommunity.com/DevCenter/setuptools) 0.6 - -Certain sub-modules have extra requirements: - -- `async` requires [Twisted](http://twistedmatrix.com/trac/) 2.5 -- `files` requires [path](http://www.jorendorff.com/articles/python/path/) 2.2 - -This library has only been tested on Linux. - -Setup ------ - -To install, run `easy_install python-commons`, or download the source tarball -and run `python setup.py install`. - -Related Work ------------- - -- [ASPN Cookbook]: a valuable repository of Python snippets -- [AIMA Utilities]: accompaniment to a popular AI textbook - -[ASPN Cookbook]: http://aspn.activestate.com/ASPN/Cookbook/Python -[AIMA Utilities]: http://aima.cs.berkeley.edu/python/utils.py - -Changes -------- - -version 0.4 - -- removed extraneous debug print statements -- added `logout()` context manager -- added `seq()`, `default_if_none()` -- fixed missing `import` bug -- released for [Mailing List - Filter](http://assorted.sf.net/mailing-list-filter/) - -version 0.3 - -- added versioned guards -- added file memoization -- added retry with exp backoff -- added `countstep()` -- released for - [gbookmark2delicious](http://gbookmark2delicious.googlecode.com/) - -version 0.2 - -- added `clients`, `setup` -- released for [icedb](http://cartel.csail.mit.edu/icedb/) - -version 0.1 - -- initial release Copied: python-commons/tags/0.5/README (from rev 812, python-commons/trunk/README) =================================================================== --- python-commons/tags/0.5/README (rev 0) +++ python-commons/tags/0.5/README 2008-05-14 18:00:58 UTC (rev 814) @@ -0,0 +1,72 @@ +[documentation](doc) + +Overview +-------- + +Python Commons is a general-purpose library for Python. To get a sense of +what it provides, please glance over the [documentation](doc). + +Requirements +------------ + +- [Python](http://python.org/) 2.5 +- [setuptools](http://peak.telecommunity.com/DevCenter/setuptools) 0.6 + +Certain sub-modules have extra requirements: + +- `async` requires [Twisted](http://twistedmatrix.com/trac/) 2.5 +- `files` requires [path](http://www.jorendorff.com/articles/python/path/) 2.2 + +This library has only been tested on Linux. + +Setup +----- + +To install, run `easy_install python-commons`, or download the source tarball +and run `python setup.py install`. + +Related Work +------------ + +- [ASPN Cookbook]: a valuable repository of Python snippets +- [AIMA Utilities]: accompaniment to a popular AI textbook + +[ASPN Cookbook]: http://aspn.activestate.com/ASPN/Cookbook/Python +[AIMA Utilities]: http://aima.cs.berkeley.edu/python/utils.py + +Changes +------- + +version 0.5, 2008-05-14 + +- added `cp1252_to_unicode()` +- made `setup()` more flexible +- released for + [gbookmark2delicious](http://gbookmark2delicious.googlecode.com/) + +version 0.4, 2008-05-08 + +- removed extraneous debug print statements +- added `logout()` context manager +- added `seq()`, `default_if_none()` +- fixed missing `import` bug +- released for [Mailing List + Filter](http://assorted.sf.net/mailing-list-filter/) + +version 0.3, 2008-04-30 + +- added versioned guards +- added file memoization +- added retry with exp backoff +- added `countstep()` +- released for + [gbookmark2delicious](http://gbookmark2delicious.googlecode.com/) + +version 0.2, 2008-02-04 + +- added `clients`, `setup` +- released for [icedb](http://cartel.csail.mit.edu/icedb/) + +version 0.1, 2007-03-24 + +- initial release Modified: python-commons/tags/0.5/publish.bash =================================================================== --- python-commons/trunk/publish.bash 2008-05-10 05:46:06 UTC (rev 791) +++ python-commons/tags/0.5/publish.bash 2008-05-14 18:00:58 UTC (rev 814) @@ -4,8 +4,11 @@ epydoc -o $stagedir/doc src/commons/ } +echo 'Remember to keep versions in sync in all three locations:' +echo '__init__.py, README (Changes), setup.bash, and setup.py' + fullname='Python Commons' -version=0.4 +version=0.5 license=psf websrcs=( README ) rels=( pypi: ) Modified: python-commons/tags/0.5/setup.py =================================================================== --- python-commons/trunk/setup.py 2008-05-10 05:46:06 UTC (rev 791) +++ python-commons/tags/0.5/setup.py 2008-05-14 18:00:58 UTC (rev 814) @@ -9,7 +9,7 @@ pkg_info_text = """ Metadata-Version: 1.1 Name: python-commons -Version: 0.4 +Version: 0.5 Author: Yang Zhang Author-email: yaaang NOSPAM at REMOVECAPS gmail Home-page: http://assorted.sourceforge.net/python-commons Modified: python-commons/tags/0.5/src/commons/__init__.py =================================================================== --- python-commons/trunk/src/commons/__init__.py 2008-05-10 05:46:06 UTC (rev 791) +++ python-commons/tags/0.5/src/commons/__init__.py 2008-05-14 18:00:58 UTC (rev 814) @@ -10,7 +10,7 @@ @license: PSF """ -__version__ = ( 0, 2, 0 ) +__version__ = ( 0, 5, 0 ) __all__ = [ 'async', 'control', 'decs', Deleted: python-commons/tags/0.5/src/commons/setup.py =================================================================== --- python-commons/trunk/src/commons/setup.py 2008-05-10 05:46:06 UTC (rev 791) +++ python-commons/tags/0.5/src/commons/setup.py 2008-05-14 18:00:58 UTC (rev 814) @@ -1,100 +0,0 @@ -#!/usr/bin/env python -# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- -# vim:ft=python:et:sw=4:ts=4 - -""" -Common code for setup.py files. Details about the Trove classifiers are -available at -U{http://pypi.python.org/pypi?%3Aaction=list_classifiers}. -""" - -arg_keys = """ -name -version -author -author_email -description: Summary -download_url: Download-url -long_description: Description -keywords: Keywords -url: Home-page -license -classifiers: Classifier -platforms: Platform -""" - -import sys -if not hasattr(sys, "version_info") or sys.version_info < (2, 3): - from distutils.core import setup - _setup = setup - def setup(**kwargs): - for key in [ - # distutils >= Python 2.3 args - # XXX probably download_url came in earlier than 2.3 - "classifiers", "download_url", - # setuptools args - "install_requires", "zip_safe", "test_suite", - ]: - if kwargs.has_key(key): - del kwargs[key] - # Only want packages keyword if this is a package, - # only want py_modules keyword if this is a single-file module, - # so get rid of packages or py_modules keyword as appropriate. - if kwargs["packages"] is None: - del kwargs["packages"] - else: - del kwargs["py_modules"] - apply(_setup, (), kwargs) -else: - from setuptools import setup, find_packages - -def run_setup( pkg_info_text, *orig_args, **orig_kwargs ): - list_keys = set( [ 'Classifier' ] ) - pkg_info = {} - for line in pkg_info_text.split( '\n' ): - if line.strip() != '': - if line.startswith( ' '*8 ): - pkg_info[ key ] += line[ 7 : ] - else: - key, value = line.split( ': ', 1 ) - if key in list_keys: - try: - pkg_info[ key ].append( value ) - except: - pkg_info[ key ] = [ value ] - else: - pkg_info[ key ] = value - - args_nontranslations = set() - args_translations = {} - for line in arg_keys.split( '\n' ): - if line.strip() != '': - splitted = line.split( ': ', 1 ) - dest_name = splitted[ 0 ] - if len( splitted ) == 2: - source_name = splitted[ 1 ] - args_translations[ source_name ] = dest_name - else: - args_nontranslations.add( dest_name ) - - args = {} - for key, value in pkg_info.iteritems(): - dest_name = None - try: - dest_name = args_translations[ key ] - except KeyError: - key = key.lower().replace('-','_') - if key in args_nontranslations: - dest_name = key - if dest_name is not None: - args[ dest_name ] = value - - # this also allows user to override our args - args.update( orig_kwargs ) - args.update( { - 'package_dir': {'':'src'}, - 'packages': find_packages('src'), - 'zip_safe': True, - } ) - - setup( *orig_args, **args ) Copied: python-commons/tags/0.5/src/commons/setup.py (from rev 805, python-commons/trunk/src/commons/setup.py) =================================================================== --- python-commons/tags/0.5/src/commons/setup.py (rev 0) +++ python-commons/tags/0.5/src/commons/setup.py 2008-05-14 18:00:58 UTC (rev 814) @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- +# vim:ft=python:et:sw=4:ts=4 + +""" +Common code for setup.py files. Details about the Trove classifiers are +available at +U{http://pypi.python.org/pypi?%3Aaction=list_classifiers}. +""" + +arg_keys = """ +name +version +author +author_email +description: Summary +download_url: Download-url +long_description: Description +keywords: Keywords +url: Home-page +license +classifiers: Classifier +platforms: Platform +""" + +import sys +if not hasattr(sys, "version_info") or sys.version_info < (2, 3): + from distutils.core import setup + _setup = setup + def setup(**kwargs): + for key in [ + # distutils >= Python 2.3 args + # XXX probably download_url came in earlier than 2.3 + "classifiers", "download_url", + # setuptools args + "install_requires", "zip_safe", "test_suite", + ]: + if kwargs.has_key(key): + del kwargs[key] + # Only want packages keyword if this is a package, + # only want py_modules keyword if this is a single-file module, + # so get rid of packages or py_modules keyword as appropriate. + if kwargs["packages"] is None: + del kwargs["packages"] + else: + del kwargs["py_modules"] + apply(_setup, (), kwargs) +else: + from setuptools import setup, find_packages + +def run_setup( pkg_info_text, srcdir = 'src', *orig_args, **orig_kwargs ): + list_keys = set( [ 'Classifier' ] ) + pkg_info = {} + for line in pkg_info_text.split( '\n' ): + if line.strip() != '': + if line.startswith( ' '*8 ): + pkg_info[ key ] += line[ 7 : ] + else: + key, value = line.split( ': ', 1 ) + if key in list_keys: + try: + pkg_info[ key ].append( value ) + except: + pkg_info[ key ] = [ value ] + else: + pkg_info[ key ] = value + + args_nontranslations = set() + args_translations = {} + for line in arg_keys.split( '\n' ): + if line.strip() != '': + splitted = line.split( ': ', 1 ) + dest_name = splitted[ 0 ] + if len( splitted ) == 2: + source_name = splitted[ 1 ] + args_translations[ source_name ] = dest_name + else: + args_nontranslations.add( dest_name ) + + args = {} + for key, value in pkg_info.iteritems(): + dest_name = None + try: + dest_name = args_translations[ key ] + except KeyError: + key = key.lower().replace('-','_') + if key in args_nontranslations: + dest_name = key + if dest_name is not None: + args[ dest_name ] = value + + # this also allows user to override our args + args.update( orig_kwargs ) + args.update( { + 'package_dir': {'':srcdir}, + 'packages': find_packages(srcdir), + 'zip_safe': True, + } ) + + setup( *orig_args, **args ) Deleted: python-commons/tags/0.5/src/commons/strs.py =================================================================== --- python-commons/trunk/src/commons/strs.py 2008-05-10 05:46:06 UTC (rev 791) +++ python-commons/tags/0.5/src/commons/strs.py 2008-05-14 18:00:58 UTC (rev 814) @@ -1,20 +0,0 @@ -# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- -# vim:ft=python:et:sw=4:ts=4 - -""" -String formatting. -""" - -import itertools - -def format( *args ): - """Formats the args as they would be by the C{print} built-in.""" - return ' '.join( itertools.imap( str, args ) ) - -def safe_ascii( s ): - """Casts a Unicode string to a regular ASCCII string. This may be - lossy.""" - if isinstance( s, unicode ) and s == str( s ): - return str( s ) - else: - return s Copied: python-commons/tags/0.5/src/commons/strs.py (from rev 804, python-commons/trunk/src/commons/strs.py) =================================================================== --- python-commons/tags/0.5/src/commons/strs.py (rev 0) +++ python-commons/tags/0.5/src/commons/strs.py 2008-05-14 18:00:58 UTC (rev 814) @@ -0,0 +1,58 @@ +# -*- mode: python; tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4; -*- +# vim:ft=python:et:sw=4:ts=4 + +""" +String formatting. +""" + +import itertools + +def format( *args ): + """Formats the args as they would be by the C{print} built-in.""" + return ' '.join( itertools.imap( str, args ) ) + +def safe_ascii( s ): + """Casts a Unicode string to a regular ASCCII string. This may be + lossy.""" + if isinstance( s, unicode ) and s == str( s ): + return str( s ) + else: + return s + +cp1252_to_unicode_translations = [ (u'\x80',u'\u20AC'), + (u'\x82',u'\u201A'), + (u'\x83',u'\u0192'), + (u'\x84',u'\u201E'), + (u'\x85',u'\u2026'), + (u'\x86',u'\u2020'), + (u'\x87',u'\u2021'), + (u'\x88',u'\u02C6'), + (u'\x89',u'\u2030'), + (u'\x8A',u'\u0160'), + (u'\x8B',u'\u2039'), + (u'\x8C',u'\u0152'), + (u'\x8E',u'\u017D'), + (u'\x91',u'\u2018'), + (u'\x92',u'\u2019'), + (u'\x93',u'\u201C'), + (u'\x94',u'\u201D'), + (u'\x95',u'\u2022'), + (u'\x96',u'\u2013'), + (u'\x97',u'\u2014'), + (u'\x98',u'\u02DC'), + (u'\x99',u'\u2122'), + (u'\x9A',u'\u0161'), + (u'\x9B',u'\u203A'), + (u'\x9C',u'\u0153'), + (u'\x9E',u'\u017E'), + (u'\x9F',u'\u0178') ] + +def cp1252_to_unicode(x): + """Converts characters 0x80 through 0x9f to their proper Unicode + equivalents. See + U{http://www.intertwingly.net/stories/2004/04/14/i18n.html} for the nice + translation table on which this is based.""" + for a,b in cp1252_to_unicode_translations: + x = x.replace(a,b) + return x + This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |