|
From: <ian...@us...> - 2003-09-07 18:07:17
|
Update of /cvsroot/sqlobject/SQLObject/tests In directory sc8-pr-cvs1:/tmp/cvs-serv20943/tests Modified Files: test.py Added Files: coverage.py Log Message: Added --coverage option to tests, which creates annotated source files that show coverage. --- NEW FILE: coverage.py --- #!/usr/bin/python # # Perforce Defect Tracking Integration Project # <http://www.ravenbrook.com/project/p4dti/> # # COVERAGE.PY -- COVERAGE TESTING # # Gareth Rees, Ravenbrook Limited, 2001-12-04 # # # 1. INTRODUCTION # # This module provides coverage testing for Python code. # # The intended readership is all Python developers. # # This document is not confidential. # # See [GDR 2001-12-04a] for the command-line interface, programmatic # interface and limitations. See [GDR 2001-12-04b] for requirements and # design. """Usage: coverage.py -x MODULE.py [ARG1 ARG2 ...] Execute module, passing the given command-line arguments, collecting coverage data. coverage.py -e Erase collected coverage data. coverage.py -r [-m] FILE1 FILE2 ... Report on the statement coverage for the given files. With the -m option, show line numbers of the statements that weren't executed. coverage.py -a [-d dir] FILE1 FILE2 ... Make annotated copies of the given files, marking statements that are executed with > and statements that are missed with !. With the -d option, make the copies in that directory. Without the -d option, make each copy in the same directory as the original. Coverage data is saved in the file .coverage by default. Set the COVERAGE_FILE environment variable to save it somewhere else.""" import os import re import string import sys import types # 2. IMPLEMENTATION # # This uses the "singleton" pattern. # # The word "morf" means a module object (from which the source file can # be deduced by suitable manipulation of the __file__ attribute) or a # filename. # # When we generate a coverage report we have to canonicalize every # filename in the coverage dictionary just in case it refers to the # module we are reporting on. It seems a shame to throw away this # information so the data in the coverage dictionary is transferred to # the 'cexecuted' dictionary under the canonical filenames. # # The coverage dictionary is called "c" and the trace function "t". The # reason for these short names is that Python looks up variables by name # at runtime and so execution time depends on the length of variables! # In the bottleneck of this application it's appropriate to abbreviate # names to increase speed. # A dictionary with an entry for (Python source file name, line number # in that file) if that line has been executed. c = {} # t(f, x, y). This method is passed to sys.settrace as a trace # function. See [van Rossum 2001-07-20b, 9.2] for an explanation of # sys.settrace and the arguments and return value of the trace function. # See [van Rossum 2001-07-20a, 3.2] for a description of frame and code # objects. def t(f, x, y): c[(f.f_code.co_filename, f.f_lineno)] = 1 return t the_coverage = None class coverage: error = "coverage error" # Name of the cache file (unless environment variable is set). cache_default = ".coverage" # Environment variable naming the cache file. cache_env = "COVERAGE_FILE" # A map from canonical Python source file name to a dictionary in # which there's an entry for each line number that has been # executed. cexecuted = {} # Cache of results of calling the analysis() method, so that you can # specify both -r and -a without doing double work. analysis_cache = {} # Cache of results of calling the canonical_filename() method, to # avoid duplicating work. canonical_filename_cache = {} def __init__(self): global the_coverage if the_coverage: raise self.error, "Only one coverage object allowed." self.cache = os.environ.get(self.cache_env, self.cache_default) self.restore() self.analysis_cache = {} def help(self, error=None): if error: print error print print __doc__ sys.exit(1) def command_line(self): import getopt settings = {} optmap = { '-a': 'annotate', '-d:': 'directory=', '-e': 'erase', '-h': 'help', '-i': 'ignore-errors', '-m': 'show-missing', '-r': 'report', '-x': 'execute', } short_opts = string.join(map(lambda o: o[1:], optmap.keys()), '') long_opts = optmap.values() options, args = getopt.getopt(sys.argv[1:], short_opts, long_opts) for o, a in options: if optmap.has_key(o): settings[optmap[o]] = 1 elif optmap.has_key(o + ':'): settings[optmap[o + ':']] = a elif o[2:] in long_opts: settings[o[2:]] = 1 elif o[2:] + '=' in long_opts: settings[o[2:]] = a else: self.help("Unknown option: '%s'." % o) if settings.get('help'): self.help() for i in ['erase', 'execute']: for j in ['annotate', 'report']: if settings.get(i) and settings.get(j): self.help("You can't specify the '%s' and '%s' " "options at the same time." % (i, j)) args_needed = (settings.get('execute') or settings.get('annotate') or settings.get('report')) action = settings.get('erase') or args_needed if not action: self.help("You must specify at least one of -e, -x, -r, " "or -a.") if not args_needed and args: self.help("Unexpected arguments %s." % args) if settings.get('erase'): self.erase() if settings.get('execute'): if not args: self.help("Nothing to do.") sys.argv = args self.start() import __main__ sys.path[0] = os.path.dirname(sys.argv[0]) execfile(sys.argv[0], __main__.__dict__) if not args: args = self.cexecuted.keys() ignore_errors = settings.get('ignore-errors') show_missing = settings.get('show-missing') directory = settings.get('directory=') if settings.get('report'): self.report(args, show_missing, ignore_errors) if settings.get('annotate'): self.annotate(args, directory, ignore_errors) def start(self): sys.settrace(t) def stop(self): sys.settrace(None) def erase(self): global c c = {} self.analysis_cache = {} self.cexecuted = {} if os.path.exists(self.cache): os.remove(self.cache) # save(). Save coverage data to the coverage cache. def save(self): self.canonicalize_filenames() cache = open(self.cache, 'wb') import marshal marshal.dump(self.cexecuted, cache) cache.close() # restore(). Restore coverage data from the coverage cache (if it # exists). def restore(self): global c c = {} self.cexecuted = {} if not os.path.exists(self.cache): return try: cache = open(self.cache, 'rb') import marshal cexecuted = marshal.load(cache) cache.close() if isinstance(cexecuted, types.DictType): self.cexecuted = cexecuted except: pass # canonical_filename(filename). Return a canonical filename for the # file (that is, an absolute path with no redundant components and # normalized case). See [GDR 2001-12-04b, 3.3]. def canonical_filename(self, filename): if not self.canonical_filename_cache.has_key(filename): f = filename if os.path.isabs(f) and not os.path.exists(f): f = os.path.basename(f) if not os.path.isabs(f): for path in [os.curdir] + sys.path: g = os.path.join(path, f) if os.path.exists(g): f = g break cf = os.path.normcase(os.path.abspath(f)) self.canonical_filename_cache[filename] = cf return self.canonical_filename_cache[filename] # canonicalize_filenames(). Copy results from "executed" to # "cexecuted", canonicalizing filenames on the way. Clear the # "executed" map. def canonicalize_filenames(self): global c for filename, lineno in c.keys(): f = self.canonical_filename(filename) if not self.cexecuted.has_key(f): self.cexecuted[f] = {} self.cexecuted[f][lineno] = 1 c = {} # morf_filename(morf). Return the filename for a module or file. def morf_filename(self, morf): if isinstance(morf, types.ModuleType): if not hasattr(morf, '__file__'): raise self.error, "Module has no __file__ attribute." file = morf.__file__ else: file = morf return self.canonical_filename(file) # analyze_morf(morf). Analyze the module or filename passed as # the argument. If the source code can't be found, raise an error. # Otherwise, return a pair of (1) the canonical filename of the # source code for the module, and (2) a list of lines of statements # in the source code. def analyze_morf(self, morf): if self.analysis_cache.has_key(morf): return self.analysis_cache[morf] filename = self.morf_filename(morf) ext = os.path.splitext(filename)[1] if ext == '.pyc': if not os.path.exists(filename[0:-1]): raise self.error, ("No source for compiled code '%s'." % filename) filename = filename[0:-1] elif ext != '.py': raise self.error, "File '%s' not Python source." % filename source = open(filename, 'r') import parser tree = parser.suite(source.read()).totuple(1) source.close() statements = {} self.find_statements(tree, statements) lines = statements.keys() lines.sort() result = filename, lines self.analysis_cache[morf] = result return result # find_statements(tree, dict). Find each statement in the parse # tree and record the line on which the statement starts in the # dictionary (by assigning it to 1). # # It works by walking the whole tree depth-first. Every time it # comes across a statement (symbol.stmt -- this includes compound # statements like 'if' and 'while') it calls find_statement, which # descends the tree below the statement to find the first terminal # token in that statement and record the lines on which that token # was found. # # This algorithm may find some lines several times (because of the # grammar production statement -> compound statement -> statement), # but that doesn't matter because we record lines as the keys of the # dictionary. # # See also [GDR 2001-12-04b, 3.2]. def find_statements(self, tree, dict): import symbol, token if token.ISNONTERMINAL(tree[0]): for t in tree[1:]: self.find_statements(t, dict) if tree[0] == symbol.stmt: self.find_statement(tree[1], dict) elif (tree[0] == token.NAME and tree[1] in ['elif', 'except', 'finally']): dict[tree[2]] = 1 def find_statement(self, tree, dict): import token while token.ISNONTERMINAL(tree[0]): tree = tree[1] dict[tree[2]] = 1 # format_lines(statements, lines). Format a list of line numbers # for printing by coalescing groups of lines as long as the lines # represent consecutive statements. This will coalesce even if # there are gaps between statements, so if statements = # [1,2,3,4,5,10,11,12,13,14] and lines = [1,2,5,10,11,13,14] then # format_lines will return "1-2, 5-11, 13-14". def format_lines(self, statements, lines): pairs = [] i = 0 j = 0 start = None pairs = [] while i < len(statements) and j < len(lines): if statements[i] == lines[j]: if start == None: start = lines[j] end = lines[j] j = j + 1 elif start: pairs.append((start, end)) start = None i = i + 1 if start: pairs.append((start, end)) def stringify(pair): start, end = pair if start == end: return "%d" % start else: return "%d-%d" % (start, end) import string return string.join(map(stringify, pairs), ", ") def analysis(self, morf): filename, statements = self.analyze_morf(morf) self.canonicalize_filenames() if not self.cexecuted.has_key(filename): self.cexecuted[filename] = {} missing = [] for line in statements: if not self.cexecuted[filename].has_key(line): missing.append(line) return (filename, statements, missing, self.format_lines(statements, missing)) def morf_name(self, morf): if isinstance(morf, types.ModuleType): return morf.__name__ else: return os.path.splitext(os.path.basename(morf))[0] def report(self, morfs, show_missing=1, ignore_errors=0): if not isinstance(morfs, types.ListType): morfs = [morfs] max_name = max([5,] + map(len, map(self.morf_name, morfs))) fmt_name = "%%- %ds " % max_name fmt_err = fmt_name + "%s: %s" header = fmt_name % "Name" + " Stmts Exec Cover" fmt_coverage = fmt_name + "% 6d % 6d % 5d%%" if show_missing: header = header + " Missing" fmt_coverage = fmt_coverage + " %s" print header print "-" * len(header) total_statements = 0 total_executed = 0 for morf in morfs: name = self.morf_name(morf) try: _, statements, missing, readable = self.analysis(morf) n = len(statements) m = n - len(missing) if n > 0: pc = 100.0 * m / n else: pc = 100.0 args = (name, n, m, pc) if show_missing: args = args + (readable,) print fmt_coverage % args total_statements = total_statements + n total_executed = total_executed + m except KeyboardInterrupt: raise except: if not ignore_errors: type, msg = sys.exc_info()[0:2] print fmt_err % (name, type, msg) if len(morfs) > 1: print "-" * len(header) if total_statements > 0: pc = 100.0 * total_executed / total_statements else: pc = 100.0 args = ("TOTAL", total_statements, total_executed, pc) if show_missing: args = args + ("",) print fmt_coverage % args # annotate(morfs, ignore_errors). blank_re = re.compile("\\s*(#|$)") else_re = re.compile("\\s*else\\s*:\\s*(#|$)") def annotate(self, morfs, directory=None, ignore_errors=0): for morf in morfs: try: filename, statements, missing, _ = self.analysis(morf) source = open(filename, 'r') if directory: dest_file = os.path.join(directory, os.path.basename(filename) + ',cover') else: dest_file = filename + ',cover' dest = open(dest_file, 'w') lineno = 0 i = 0 j = 0 covered = 1 while 1: line = source.readline() if line == '': break lineno = lineno + 1 while i < len(statements) and statements[i] < lineno: i = i + 1 while j < len(missing) and missing[j] < lineno: j = j + 1 if i < len(statements) and statements[i] == lineno: covered = j >= len(missing) or missing[j] > lineno if self.blank_re.match(line): dest.write(' ') elif self.else_re.match(line): # Special logic for lines containing only # 'else:'. See [GDR 2001-12-04b, 3.2]. if i >= len(statements) and j >= len(missing): dest.write('! ') elif i >= len(statements) or j >= len(missing): dest.write('> ') elif statements[i] == missing[j]: dest.write('! ') else: dest.write('> ') elif covered: dest.write('> ') else: dest.write('! ') dest.write(line) source.close() dest.close() except KeyboardInterrupt: raise except: if not ignore_errors: raise # Singleton object. the_coverage = coverage() # Module functions call methods in the singleton object. def start(*args): return apply(the_coverage.start, args) def stop(*args): return apply(the_coverage.stop, args) def erase(*args): return apply(the_coverage.erase, args) def analysis(*args): return apply(the_coverage.analysis, args) def report(*args): return apply(the_coverage.report, args) # Save coverage data when Python exits. (The atexit module wasn't # introduced until Python 2.0, so use sys.exitfunc when it's not # available.) try: import atexit atexit.register(the_coverage.save) except ImportError: sys.exitfunc = the_coverage.save # Command-line interface. if __name__ == '__main__': the_coverage.command_line() # A. REFERENCES # # [GDR 2001-12-04a] "Statement coverage for Python"; Gareth Rees; # Ravenbrook Limited; 2001-12-04; # <http://www.garethrees.org/2001/12/04/python-coverage/>. # # [GDR 2001-12-04b] "Statement coverage for Python: design and # analysis"; Gareth Rees; Ravenbrook Limited; 2001-12-04; # <http://www.garethrees.org/2001/12/04/python-coverage/design.html>. # # [van Rossum 2001-07-20a] "Python Reference Manual (releae 2.1.1)"; # Guide van Rossum; 2001-07-20; # <http://www.python.org/doc/2.1.1/ref/ref.html>. # # [van Rossum 2001-07-20b] "Python Library Reference"; Guido van Rossum; # 2001-07-20; <http://www.python.org/doc/2.1.1/lib/lib.html>. # # # B. DOCUMENT HISTORY # # 2001-12-04 GDR Created. # # 2001-12-06 GDR Added command-line interface and source code # annotation. # # 2001-12-09 GDR Moved design and interface to separate documents. # # 2001-12-10 GDR Open cache file as binary on Windows. Allow # simultaneous -e and -x, or -a and -r. # # 2001-12-12 GDR Added command-line help. Cache analysis so that it # only needs to be done once when you specify -a and -r. # # 2001-12-13 GDR Improved speed while recording. Portable between # Python 1.5.2 and 2.1.1. # # 2002-01-03 GDR Module-level functions work correctly. # # 2002-01-07 GDR Update sys.path when running a file with the -x option, # so that it matches the value the program would get if it were run on # its own. # # # C. COPYRIGHT AND LICENCE # # Copyright 2001 Gareth Rees. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the # distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS # OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH # DAMAGE. # # # # $Id: coverage.py,v 1.1 2003/09/07 18:07:11 ianbicking Exp $ Index: test.py =================================================================== RCS file: /cvsroot/sqlobject/SQLObject/tests/test.py,v retrieving revision 1.24 retrieving revision 1.25 diff -C2 -d -r1.24 -r1.25 *** test.py 7 Sep 2003 07:17:50 -0000 1.24 --- test.py 7 Sep 2003 18:07:11 -0000 1.25 *************** *** 7,14 **** """ from SQLObjectTest import * from SQLObject import * from mx import DateTime - from __future__ import generators ######################################## --- 7,22 ---- """ + from __future__ import generators + + import sys + if '--coverage' in sys.argv: + import coverage + print 'Starting coverage' + coverage.erase() + coverage.start() + from SQLObjectTest import * from SQLObject import * from mx import DateTime ######################################## *************** *** 620,627 **** ######################################## if __name__ == '__main__': ! import unittest, sys dbs = [] newArgs = [] for arg in sys.argv[1:]: if arg.startswith('-d'): --- 628,706 ---- ######################################## + def coverModules(): + sys.stdout.write('Writing coverage...') + sys.stdout.flush() + here = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + from SQLObject import DBConnection as tmp + there = os.path.dirname(os.path.abspath(tmp.__file__)) + for name, mod in sys.modules.items(): + if not mod: + continue + try: + modFile = os.path.abspath(mod.__file__) + except AttributeError: + # Probably a C extension + continue + if modFile.startswith(here) or modFile.startswith(there): + writeCoverage(mod, there, os.path.join(here, 'SQLObject')) + sys.stdout.write('done.\n') + + + def writeCoverage(module, oldBase, newBase): + filename, numbers, unexecuted, s = coverage.analysis(module) + coverFilename = filename + ',cover' + if coverFilename.startswith(oldBase): + coverFilename = newBase + coverFilename[len(oldBase):] + fout = open(coverFilename, 'w') + fin = open(filename) + i = 1 + lines = 0 + good = 0 + while 1: + line = fin.readline() + if not line: break + assert line[-1] == '\n' + fout.write(line[:-1]) + unused = i in unexecuted + interesting = interestingLine(line, unused) + if interesting: + if unused: + fout.write(' '*(72-len(line))) + fout.write('#@@@@') + lastUnused = True + else: + lastUnused = False + good += 1 + lines += 1 + fout.write('\n') + i += 1 + fout.write('\n# Coverage:\n') + fout.write('# %i/%i, %i%%' % ( + good, lines, lines and int(good*100/lines))) + fout.close() + fin.close() + + def interestingLine(line, unused): + line = line.strip() + if not line: + return False + if line.startswith('#'): + return False + if line in ('"""', '"""'): + return False + if line.startswith('global '): + return False + if line.startswith('def ') and not unused: + # If a def *isn't* executed, that's interesting + return False + if line.startswith('class ') and not unused: + return False + return True + if __name__ == '__main__': ! import unittest, sys, os dbs = [] newArgs = [] + doCoverage = False for arg in sys.argv[1:]: if arg.startswith('-d'): *************** *** 641,644 **** --- 720,727 ---- SQLObjectTest.debugInserts = True continue + if arg in ('--coverage',): + # Handled earlier, so we get better coverage + doCoverage = True + continue newArgs.append(arg) sys.argv = [sys.argv[0]] + newArgs *************** *** 654,655 **** --- 737,742 ---- except SystemExit: pass + if doCoverage: + coverage.stop() + coverModules() + |
