Menu
▾
▴
[Jython-checkins] SF.net SVN: jython: [4885] branches/asm/Lib/test
|
From: <pj...@us...> - 2008-07-11 03:46:16
|
Revision: 4885
http://jython.svn.sourceforge.net/jython/?rev=4885&view=rev
Author: pjenvey
Date: 2008-07-10 20:46:09 -0700 (Thu, 10 Jul 2008)
Log Message:
-----------
add a -j option to regrtest for capturing test output to JUnit XML files (one
file per TestCase or module import or doctest) in the specified dir
Modified Paths:
--------------
branches/asm/Lib/test/regrtest.py
branches/asm/Lib/test/test_support.py
Added Paths:
-----------
branches/asm/Lib/test/junit_xml.py
Added: branches/asm/Lib/test/junit_xml.py
===================================================================
--- branches/asm/Lib/test/junit_xml.py (rev 0)
+++ branches/asm/Lib/test/junit_xml.py 2008-07-11 03:46:09 UTC (rev 4885)
@@ -0,0 +1,278 @@
+"""Support for writing JUnit XML test results for the regrtest"""
+import os
+import re
+import sys
+import time
+import traceback
+import unittest
+from StringIO import StringIO
+from xml.sax import saxutils
+
+# Invalid XML characters (control chars)
+EVIL_CHARACTERS_RE = re.compile(r"[\000-\010\013\014\016-\037]")
+
+class JUnitXMLTestRunner:
+
+ """A unittest runner that writes results to a JUnit XML file in
+ xml_dir
+ """
+
+ def __init__(self, xml_dir):
+ self.xml_dir = xml_dir
+
+ def run(self, test):
+ result = JUnitXMLTestResult(self.xml_dir)
+ test(result)
+ result.write_xml()
+ return result
+
+
+class JUnitXMLTestResult(unittest.TestResult):
+
+ """JUnit XML test result writer.
+
+ The name of the file written to is determined from the full module
+ name of the first test ran
+ """
+
+ def __init__(self, xml_dir):
+ unittest.TestResult.__init__(self)
+ self.xml_dir = xml_dir
+
+ # The module name of the first test ran
+ self.module_name = None
+
+ # All TestCases
+ self.tests = []
+
+ # Start time
+ self.start = None
+
+ self.old_stdout = sys.stdout
+ self.old_stderr = sys.stderr
+ sys.stdout = self.stdout = Tee(sys.stdout)
+ sys.stderr = self.stderr = Tee(sys.stderr)
+
+ def startTest(self, test):
+ unittest.TestResult.startTest(self, test)
+ self.ensure_module_name(test)
+ self.error, self.failure = None, None
+ self.start = time.time()
+
+ def stopTest(self, test):
+ took = time.time() - self.start
+ unittest.TestResult.stopTest(self, test)
+ args = [test, took]
+ if self.error:
+ args.extend(['error', self.error])
+ elif self.failure:
+ args.extend(['failure', self.failure])
+ self.tests.append(TestInfo.from_testcase(*args))
+
+ def addError(self, test, err):
+ unittest.TestResult.addError(self, test, err)
+ self.error = err
+
+ def addFailure(self, test, err):
+ unittest.TestResult.addFailure(self, test, err)
+ self.failure = err
+
+ def ensure_module_name(self, test):
+ """Set self.module_name from test if not already set"""
+ if not self.module_name:
+ self.module_name = '.'.join(test.id().split('.')[:-1])
+
+ def write_xml(self):
+ if not self.module_name:
+ # No tests ran, nothing to write
+ return
+ took = time.time() - self.start
+
+ stdout = self.stdout.getvalue()
+ stderr = self.stderr.getvalue()
+ sys.stdout = self.old_stdout
+ sys.stderr = self.old_stderr
+
+ ensure_dir(self.xml_dir)
+ filename = os.path.join(self.xml_dir, 'TEST-%s.xml' % self.module_name)
+ stream = open(filename, 'w')
+
+ write_testsuite_xml(stream, len(self.tests), len(self.errors),
+ len(self.failures), 0, self.module_name, took)
+
+ for info in self.tests:
+ info.write_xml(stream)
+
+ write_stdouterr_xml(stream, stdout, stderr)
+
+ stream.write('</testsuite>')
+ stream.close()
+
+
+class TestInfo(object):
+
+ """The JUnit XML <testcase/> model."""
+
+ def __init__(self, name, took, type=None, exc_info=None):
+ # The name of the test
+ self.name = name
+
+ # How long it took
+ self.took = took
+
+ # Type of test: 'error', 'failure' 'skipped', or None for a
+ # success
+ self.type = type
+
+ if exc_info:
+ self.exc_name = exc_name(exc_info)
+ self.message = exc_message(exc_info)
+ self.traceback = safe_str(''.join(
+ traceback.format_exception(*exc_info)))
+ else:
+ self.exc_name = self.message = self.traceback = ''
+
+ @classmethod
+ def from_testcase(cls, testcase, took, type=None, exc_info=None):
+ name = testcase.id().split('.')[-1]
+ return cls(name, took, type, exc_info)
+
+ def write_xml(self, stream):
+ stream.write(' <testcase name="%s" time="%.3f"' % (self.name,
+ self.took))
+
+ if not self.type:
+ # test was successful
+ stream.write('/>\n')
+ return
+
+ stream.write('>\n <%s type="%s" message=%s><![CDATA[%s]]></%s>\n' %
+ (self.type, self.exc_name, saxutils.quoteattr(self.message),
+ escape_cdata(self.traceback), self.type))
+ stream.write(' </testcase>\n')
+
+
+class Tee(StringIO):
+
+ """Writes data to this StringIO and a separate stream"""
+
+ def __init__(self, stream):
+ StringIO.__init__(self)
+ self.stream = stream
+
+ def write(self, data):
+ StringIO.write(self, data)
+ self.stream.write(data)
+
+ def flush(self):
+ StringIO.flush(self)
+ self.stream.flush()
+
+
+def write_testsuite_xml(stream, tests, errors, failures, skipped, name, took):
+ """Write the XML header (<testsuite/>)"""
+ stream.write('<?xml version="1.0" encoding="utf-8"?>\n')
+ stream.write('<testsuite tests="%d" errors="%d" failures="%d" ' %
+ (tests, errors, failures))
+ stream.write('skipped="%d" name="%s" time="%.3f">\n' % (skipped, name,
+ took))
+
+def write_stdouterr_xml(stream, stdout, stderr):
+ """Write the stdout/err tags"""
+ if stdout:
+ stream.write(' <system-out><![CDATA[%s]]></system-out>\n' %
+ escape_cdata(safe_str(stdout)))
+ if stderr:
+ stream.write(' <system-err><![CDATA[%s]]></system-err>\n' %
+ escape_cdata(safe_str(stderr)))
+
+
+def write_direct_test(junit_xml_dir, name, took, type=None, exc_info=None,
+ stdout=None, stderr=None):
+ """Write XML for a regrtest 'direct' test; a test which was ran on
+ import (which we label as __main__.__import__)
+ """
+ return write_manual_test(junit_xml_dir, '%s.__main__' % name, '__import__',
+ took, type, exc_info, stdout, stderr)
+
+
+def write_doctest(junit_xml_dir, name, took, type=None, exc_info=None,
+ stdout=None, stderr=None):
+ """Write XML for a regrtest doctest, labeled as __main__.__doc__"""
+ return write_manual_test(junit_xml_dir, '%s.__main__' % name, '__doc__',
+ took, type, exc_info, stdout, stderr)
+
+
+def write_manual_test(junit_xml_dir, module_name, test_name, took, type=None,
+ exc_info=None, stdout=None, stderr=None):
+ """Manually write XML for one test, outside of unittest"""
+ errors = type == 'error' and 1 or 0
+ failures = type == 'failure' and 1 or 0
+ skipped = type == 'skipped' and 1 or 0
+
+ ensure_dir(junit_xml_dir)
+ stream = open(os.path.join(junit_xml_dir, 'TEST-%s.xml' % module_name),
+ 'w')
+
+ write_testsuite_xml(stream, 1, errors, failures, skipped, module_name,
+ took)
+
+ info = TestInfo(test_name, took, type, exc_info)
+ info.write_xml(stream)
+
+ write_stdouterr_xml(stream, stdout, stderr)
+
+ stream.write('</testsuite>')
+ stream.close()
+
+
+def ensure_dir(dir):
+ """Ensure dir exists"""
+ if not os.path.exists(dir):
+ os.mkdir(dir)
+
+
+def exc_name(exc_info):
+ """Determine the full name of the exception that caused exc_info"""
+ exc = exc_info[1]
+ name = getattr(exc.__class__, '__module__', '')
+ if name:
+ name += '.'
+ return name + exc.__class__.__name__
+
+
+def exc_message(exc_info):
+ """Safely return a short message passed through safe_str describing
+ exc_info, being careful of unicode values.
+ """
+ exc = exc_info[1]
+ if exc is None:
+ return safe_str(exc_info[0])
+ if isinstance(exc, BaseException) and isinstance(exc.message, unicode):
+ return safe_str(exc.message)
+ try:
+ return safe_str(str(exc))
+ except UnicodeEncodeError:
+ try:
+ val = unicode(exc)
+ return safe_str(val)
+ except UnicodeDecodeError:
+ return '?'
+
+
+def escape_cdata(cdata):
+ """Escape a string for an XML CDATA section"""
+ return cdata.replace(']]>', ']]>]]><![CDATA[')
+
+
+def safe_str(base):
+ """Return a str valid for UTF-8 XML from a basestring"""
+ if isinstance(base, unicode):
+ return remove_evil(base.encode('utf-8', 'replace'))
+ return remove_evil(base.decode('utf-8', 'replace').encode('utf-8',
+ 'replace'))
+
+
+def remove_evil(string):
+ """Remove control characters from a string"""
+ return EVIL_CHARACTERS_RE.sub('?', string)
Modified: branches/asm/Lib/test/regrtest.py
===================================================================
--- branches/asm/Lib/test/regrtest.py 2008-07-11 03:36:29 UTC (rev 4884)
+++ branches/asm/Lib/test/regrtest.py 2008-07-11 03:46:09 UTC (rev 4885)
@@ -16,6 +16,7 @@
-s: single -- run only a single test (see below)
-r: random -- randomize test execution order
-m: memo -- save results to file
+-j: junit-xml -- save results as JUnit XML to files in directory
-f: fromfile -- read names of tests to run from a file (see below)
-l: findleaks -- if GC is available detect tests that leak memory
-u: use -- specify which special resource intensive tests to run
@@ -131,6 +132,7 @@
import re
import cStringIO
import traceback
+import time
# I see no other way to suppress these warnings;
# putting them in test_grammar.py has no effect:
@@ -181,7 +183,8 @@
def main(tests=None, testdir=None, verbose=0, quiet=False, generate=False,
exclude=False, single=False, randomize=False, fromfile=None,
findleaks=False, use_resources=None, trace=False, coverdir='coverage',
- runleaks=False, huntrleaks=False, verbose2=False, expected=False, memo=None):
+ runleaks=False, huntrleaks=False, verbose2=False, expected=False,
+ memo=None, junit_xml=None):
"""Execute a test suite.
This also parses command-line options and modifies its behavior
@@ -206,7 +209,7 @@
test_support.record_original_stdout(sys.stdout)
try:
- opts, args = getopt.getopt(sys.argv[1:], 'hvgqxsrf:lu:t:TD:NLR:wM:em:',
+ opts, args = getopt.getopt(sys.argv[1:], 'hvgqxsrf:lu:t:TD:NLR:wM:emj:',
['help', 'verbose', 'quiet', 'generate',
'exclude', 'single', 'random', 'fromfile',
'findleaks', 'use=', 'threshold=', 'trace',
@@ -251,6 +254,8 @@
runleaks = True
elif o in ('-m', '--memo'):
memo = a
+ elif o in ('-j', '--junit-xml'):
+ junit_xml = a
elif o in ('-t', '--threshold'):
import gc
gc.set_threshold(int(a))
@@ -365,6 +370,7 @@
trace=False, count=True)
test_support.verbose = verbose # Tell tests to be moderately quiet
test_support.use_resources = use_resources
+ test_support.junit_xml_dir = junit_xml
save_modules = sys.modules.keys()
skips = _ExpectedSkips()
failures = _ExpectedFailures()
@@ -382,7 +388,7 @@
else:
try:
ok = runtest(test, generate, verbose, quiet, testdir,
- huntrleaks)
+ huntrleaks, junit_xml)
except KeyboardInterrupt:
# print a newline separate from the ^C
print
@@ -506,7 +512,8 @@
tests.sort()
return stdtests + tests
-def runtest(test, generate, verbose, quiet, testdir=None, huntrleaks=False):
+def runtest(test, generate, verbose, quiet, testdir=None, huntrleaks=False,
+ junit_xml=None):
"""Run a single test.
test -- the name of the test
@@ -526,12 +533,12 @@
try:
return runtest_inner(test, generate, verbose, quiet, testdir,
- huntrleaks)
+ huntrleaks, junit_xml)
finally:
cleanup_test_droppings(test, verbose)
def runtest_inner(test, generate, verbose, quiet,
- testdir=None, huntrleaks=False):
+ testdir=None, huntrleaks=False, junit_xml_dir=None):
test_support.unload(test)
if not testdir:
testdir = findtestdir()
@@ -544,6 +551,12 @@
try:
save_stdout = sys.stdout
+ if junit_xml_dir:
+ from test.junit_xml import Tee, write_direct_test
+ indirect_test = None
+ save_stderr = sys.stderr
+ sys.stdout = stdout = Tee(sys.stdout)
+ sys.stderr = stderr = Tee(sys.stderr)
try:
if cfp:
sys.stdout = cfp
@@ -553,6 +566,7 @@
else:
# Always import it from the test package
abstest = 'test.' + test
+ start = time.time()
the_package = __import__(abstest, globals(), locals(), [])
the_module = getattr(the_package, test)
# Most tests run to completion simply as a side-effect of
@@ -562,25 +576,46 @@
indirect_test = getattr(the_module, "test_main", None)
if indirect_test is not None:
indirect_test()
+ elif junit_xml_dir:
+ write_direct_test(junit_xml_dir, abstest, time.time() - start,
+ stdout=stdout.getvalue(),
+ stderr=stderr.getvalue())
if huntrleaks:
dash_R(the_module, test, indirect_test, huntrleaks)
finally:
sys.stdout = save_stdout
+ if junit_xml_dir:
+ sys.stderr = save_stderr
except test_support.ResourceDenied, msg:
if not quiet:
print test, "skipped --", msg
sys.stdout.flush()
+ if junit_xml_dir:
+ write_direct_test(junit_xml_dir, abstest, time.time() - start,
+ 'skipped', sys.exc_info(),
+ stdout=stdout.getvalue(),
+ stderr=stderr.getvalue())
return -2
except (ImportError, test_support.TestSkipped), msg:
if not quiet:
print test, "skipped --", msg
sys.stdout.flush()
+ if junit_xml_dir:
+ write_direct_test(junit_xml_dir, abstest, time.time() - start,
+ 'skipped', sys.exc_info(),
+ stdout=stdout.getvalue(),
+ stderr=stderr.getvalue())
return -1
except KeyboardInterrupt:
raise
except test_support.TestFailed, msg:
print "test", test, "failed --", msg
sys.stdout.flush()
+ if junit_xml_dir and indirect_test is None:
+ write_direct_test(junit_xml_dir, abstest, time.time() - start,
+ 'failure', sys.exc_info(),
+ stdout=stdout.getvalue(),
+ stderr=stderr.getvalue())
return 0
except:
type, value = sys.exc_info()[:2]
@@ -589,6 +624,11 @@
if verbose:
traceback.print_exc(file=sys.stdout)
sys.stdout.flush()
+ if junit_xml_dir and indirect_test is None:
+ write_direct_test(junit_xml_dir, abstest, time.time() - start,
+ 'error', sys.exc_info(),
+ stdout=stdout.getvalue(),
+ stderr=stderr.getvalue())
return 0
else:
if not cfp:
Modified: branches/asm/Lib/test/test_support.py
===================================================================
--- branches/asm/Lib/test/test_support.py 2008-07-11 03:36:29 UTC (rev 4884)
+++ branches/asm/Lib/test/test_support.py 2008-07-11 03:46:09 UTC (rev 4885)
@@ -4,6 +4,7 @@
raise ImportError, 'test_support must be imported from the test package'
import sys
+import time
class Error(Exception):
"""Base class for regression test exceptions."""
@@ -31,6 +32,7 @@
verbose = 1 # Flag set to 0 by regrtest.py
use_resources = None # Flag set to [] by regrtest.py
+junit_xml_dir = None # Option set by regrtest.py
max_memuse = 0 # Disable bigmem tests (they will still be run with
# small sizes, to make sure they work.)
@@ -414,8 +416,30 @@
def run_suite(suite, testclass=None):
+ """Run all TestCases in their own individual TestSuite"""
+ if not junit_xml_dir:
+ # Splitting tests apart slightly changes the handling of the
+ # TestFailed message
+ return _run_suite(suite, testclass)
+
+ failed = False
+ for test in suite:
+ suite = unittest.TestSuite()
+ suite.addTest(test)
+ try:
+ _run_suite(suite, testclass)
+ except TestFailed, e:
+ if not failed:
+ failed = e
+ if failed:
+ raise failed
+
+def _run_suite(suite, testclass=None):
"""Run tests from a unittest.TestSuite-derived class."""
- if verbose:
+ if junit_xml_dir:
+ from junit_xml import JUnitXMLTestRunner
+ runner = JUnitXMLTestRunner(junit_xml_dir)
+ elif verbose:
runner = unittest.TextTestRunner(sys.stdout, verbosity=2)
else:
runner = BasicTestRunner()
@@ -473,12 +497,36 @@
# output shouldn't be compared by regrtest.
save_stdout = sys.stdout
sys.stdout = get_original_stdout()
+
+ if junit_xml_dir:
+ from junit_xml import Tee, write_doctest
+ save_stderr = sys.stderr
+ sys.stdout = stdout = Tee(sys.stdout)
+ sys.stderr = stderr = Tee(sys.stderr)
+
try:
- f, t = doctest.testmod(module, verbose=verbosity)
+ start = time.time()
+ try:
+ f, t = doctest.testmod(module, verbose=verbosity)
+ except:
+ took = time.time() - start
+ if junit_xml_dir:
+ write_doctest(junit_xml_dir, module.__name__, took, 'error',
+ sys.exc_info(), stdout.getvalue(),
+ stderr.getvalue())
+ raise
+ took = time.time() - start
if f:
+ if junit_xml_dir:
+ write_doctest(junit_xml_dir, module.__name__, took, 'failure',
+ stdout=stdout.getvalue(),
+ stderr=stderr.getvalue())
raise TestFailed("%d of %d doctests failed" % (f, t))
finally:
sys.stdout = save_stdout
+ if junit_xml_dir:
+ write_doctest(junit_xml_dir, module.__name__, took,
+ stdout=stdout.getvalue(), stderr=stderr.getvalue())
if verbose:
print 'doctest (%s) ... %d tests with zero failures' % (module.__name__, t)
return f, t
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
