#6 Test runner should highlight inconsistent tests

[Anonymous]

e.g. like this:

# log the summary output to the console
if printSummary:
testFailCount = {}
testFailList = {}
log.critical("")
if self.threads > 1:
log.critical("Test duration (absolute): %.2f (secs)", time.time() - startTime)
log.critical("Test duration (additive): %.2f (secs)", self.duration)
else:
log.critical("Test duration: %.2f (secs)", time.time() - startTime)
log.critical("")
log.critical("Summary of non passes: ")
fails = 0
for cycle in self.results.keys():
for outcome in self.results[cycle].keys():
if outcome in FAILS : fails = fails + len(self.results[cycle][outcome])
if fails == 0:
log.critical(" THERE WERE NO NON PASSES")
else:
if len(self.results) == 1:
for outcome in FAILS:
for id in self.results[0][outcome]: log.critical(" %s: %s ", LOOKUP[outcome], id)
else:
for key in self.results.keys():
for outcome in FAILS:
for id in self.results[key][outcome]:
if not testFailCount.has_key(id):
testFailCount[id] = 0
testFailList[id] = []
testFailCount[id] = testFailCount[id] + 1
testFailList[id].append(key+1)
log.critical(" [CYCLE %d] %s: %s ", key+1, LOOKUP[outcome], id)

log.critical("Summary of inconsistant test results: ")
numRuns = len(self.results.keys())
for id in testFailCount:
if testFailCount[id] != numRuns:
log.critical(" Test %s failed %d/%d runs"%(id,testFailCount[id],numRuns))
log.critical(" Failed runs: %s"%testFailList[id])

[Ben Spiller]

Not a clear enough description. There shouldn't be any inconsistent results