[Aqsis-commits] SF.net SVN: aqsis:[2403] trunk/testing/regression
Brought to you by:
ltatkinson,
pgregory
Menu
▾
▴
[Aqsis-commits] SF.net SVN: aqsis:[2403] trunk/testing/regression
|
From: <mb...@us...> - 2008-08-25 08:25:17
|
Revision: 2403
http://aqsis.svn.sourceforge.net/aqsis/?rev=2403&view=rev
Author: mbaas
Date: 2008-08-25 08:25:26 +0000 (Mon, 25 Aug 2008)
Log Message:
-----------
Added a new type of job/task to run the shader compiler and do a textual comparison of the output instead of rendering an image.
Modified Paths:
--------------
trunk/testing/regression/jobs.cfg
trunk/testing/regression/rendertester.py
trunk/testing/regression/tasks.cfg
Added Paths:
-----------
trunk/testing/regression/ShaderCompilerTests/
trunk/testing/regression/ShaderCompilerTests/jobs.cfg
trunk/testing/regression/ShaderCompilerTests/missing_semicolon.sl
trunk/testing/regression/ShaderCompilerTests/plastic.sl
trunk/testing/regression/referenceShaders/
trunk/testing/regression/referenceShaders/plastic.slx
Added: trunk/testing/regression/ShaderCompilerTests/jobs.cfg
===================================================================
--- trunk/testing/regression/ShaderCompilerTests/jobs.cfg (rev 0)
+++ trunk/testing/regression/ShaderCompilerTests/jobs.cfg 2008-08-25 08:25:26 UTC (rev 2403)
@@ -0,0 +1,25 @@
+######################################################################
+# Command line jobs
+######################################################################
+
+SLCompilerJob(
+ workingdir = "",
+ shader = "plastic.sl",
+ returnCode = 0,
+ stderr = "",
+ description = "Standard plastic shader."
+)
+
+SLCompilerJob(
+ workingdir = "",
+ shader = "missing_semicolon.sl",
+ returnCode = -1,
+ stdout = "",
+ stderr =
+"""08/11/2008 22:45:07 ERROR: missing_semicolon.sl : 5 : syntax error
+08/11/2008 22:45:07 ERROR: Shader not compiled""",
+ outputFiles = None,
+ description = "A shader with a missing semicolon."
+)
+
+
Added: trunk/testing/regression/ShaderCompilerTests/missing_semicolon.sl
===================================================================
--- trunk/testing/regression/ShaderCompilerTests/missing_semicolon.sl (rev 0)
+++ trunk/testing/regression/ShaderCompilerTests/missing_semicolon.sl 2008-08-25 08:25:26 UTC (rev 2403)
@@ -0,0 +1,6 @@
+// A shader with a missing semicolon
+surface missing_semicolon()
+{
+ Ci = color "rgb" (1,0,0)
+ Oi = 1;
+}
Added: trunk/testing/regression/ShaderCompilerTests/plastic.sl
===================================================================
--- trunk/testing/regression/ShaderCompilerTests/plastic.sl (rev 0)
+++ trunk/testing/regression/ShaderCompilerTests/plastic.sl 2008-08-25 08:25:26 UTC (rev 2403)
@@ -0,0 +1,22 @@
+/* plastic.sl - Standard plastic surface for RenderMan Interface.
+ * (c) Copyright 1988, Pixar.
+ *
+ * The RenderMan (R) Interface Procedures and RIB Protocol are:
+ * Copyright 1988, 1989, Pixar. All rights reserved.
+ * RenderMan (R) is a registered trademark of Pixar.
+ */
+
+
+surface
+plastic (float Ka = 1;
+ float Kd = .5;
+ float Ks = .5;
+ float roughness = .1;
+ color specularcolor = 1;)
+{
+ normal Nf = faceforward (normalize(N),I);
+ Oi = Os;
+ Ci = Os * ( Cs * (Ka*ambient() + Kd*diffuse(Nf)) +
+ specularcolor * Ks*specular(Nf,-normalize(I),roughness));
+}
+
Modified: trunk/testing/regression/jobs.cfg
===================================================================
--- trunk/testing/regression/jobs.cfg 2008-08-24 03:44:03 UTC (rev 2402)
+++ trunk/testing/regression/jobs.cfg 2008-08-25 08:25:26 UTC (rev 2403)
@@ -42,7 +42,9 @@
# Recursively read all "jobs.cfg" config files in the "RIBs" tree...
include_cfg("RIBs/jobs.cfg", recursive=True)
+include_cfg("ShaderCompilerTests/jobs.cfg", recursive=True)
+
######################################################################
# The below jobs have to be reviewed and moved to an appropriate
# destination directory (after adding a description)
Added: trunk/testing/regression/referenceShaders/plastic.slx
===================================================================
--- trunk/testing/regression/referenceShaders/plastic.slx (rev 0)
+++ trunk/testing/regression/referenceShaders/plastic.slx 2008-08-25 08:25:26 UTC (rev 2403)
@@ -0,0 +1,63 @@
+surface
+AQSIS_V 1.4.0
+
+
+segment Data
+
+USES 460807
+
+param uniform float Ka
+param uniform float Kd
+param uniform float Ks
+param uniform float roughness
+param uniform color specularcolor
+varying normal Nf
+
+
+segment Init
+ pushif 1
+ pop Ka
+ pushif 0.5
+ pop Kd
+ pushif 0.5
+ pop Ks
+ pushif 0.1
+ pop roughness
+ pushif 1
+ setfc
+ pop specularcolor
+
+
+segment Code
+ pushv I
+ pushv N
+ normalize
+ faceforward
+ pop Nf
+ pushv Os
+ pop Oi
+ pushv roughness
+ pushv I
+ normalize
+ negp
+ pushv Nf
+ specular
+ pushv Ks
+ setfc
+ pushv specularcolor
+ mulcc
+ mulcc
+ pushv Nf
+ diffuse
+ pushv Kd
+ mulfc
+ ambient
+ pushv Ka
+ mulfc
+ addcc
+ pushv Cs
+ mulcc
+ addcc
+ pushv Os
+ mulcc
+ pop Ci
Modified: trunk/testing/regression/rendertester.py
===================================================================
--- trunk/testing/regression/rendertester.py 2008-08-24 03:44:03 UTC (rev 2402)
+++ trunk/testing/regression/rendertester.py 2008-08-25 08:25:26 UTC (rev 2403)
@@ -1,6 +1,6 @@
#!/usr/bin/env python
######################################################################
-# Renderer tester v1.15
+# Renderer tester v1.16
# Copyright 2003 Matthias Baas (mb...@us...)
######################################################################
# For a short usage description call the script with the option -h or
@@ -11,7 +11,7 @@
# executed (the output goes into the directory "html").
######################################################################
-import sys, os, os.path, time, string, shutil, copy, fnmatch, gzip, stat
+import sys, os, os.path, time, string, shutil, copy, fnmatch, gzip, stat, re, difflib
import optparse
import Queue, threading
import subprocess
@@ -30,6 +30,8 @@
# A list of all available jobs (RenderJob objects)
AllJobs = []
+# A list of all available command jobs
+AllCmdJobs = []
# A list of all the tasks to do (Task objects)
Tasks = []
# A list of all renderer definitions (Renderer objects)
@@ -166,7 +168,7 @@
# Was the job excluded?
for pattern in ExcludeRIBs:
- if fnmatch.fnmatch(job.rib, pattern):
+ if fnmatch.fnmatch(job.name, pattern):
return False
# Include everything?
@@ -175,7 +177,7 @@
# Was the job included?
for pattern in IncludeRIBs:
- if fnmatch.fnmatch(job.rib, pattern):
+ if fnmatch.fnmatch(job.name, pattern):
return True
return False
@@ -295,6 +297,72 @@
######################################################################
+# SLCompilerJob
+class SLCompilerJob:
+ """Shading compiler job.
+
+ The job contains the data required for running the shader compiler
+ and comparing its output with the expected output.
+ """
+
+ def __init__(self,
+ workingdir = "",
+ shader = "",
+ returnCode = None,
+ stdout = None,
+ stderr = None,
+ outputFiles = "_auto",
+ description = ""):
+ """Constructor.
+
+ workingdir: The working directory for running the shader compiler
+ shader: The name of the shader source file
+ returnCode: The expected return code or None
+ stdout: The expected output to stdout (as a single string) or None
+ stderr: The expected output to stderr (as a single string) or None
+ outputFiles: A single string or a list of strings containing the
+ generated output (text) files. Each file will be compared with
+ a reference file.
+ If set to None (or an empty list), no output files will be compared.
+ The default is a single name which is based on the shader name
+ (where .sl is replaced by .slx).
+ description: A description of the test
+ """
+ global AllCmdJobs, CurrentConfigDir
+
+ if not isinstance(workingdir, basestring):
+ raise ValueError("SLCompilerJob: workingdir argument must be a string")
+ if not isinstance(shader, basestring):
+ raise ValueError("SLCompilerJob: shader argument must be a string")
+ if not isinstance(description, basestring):
+ raise ValueError("SLCompilerJob: description argument must be a string")
+
+ self.workingdir = os.path.abspath(os.path.join(CurrentConfigDir[-1], workingdir))
+ self.shader = os.path.join(self.workingdir, shader)
+ self.name = shader
+ self.returnCode = returnCode
+ self.stdoutData = stdout
+ self.stderrData = stderr
+ self.description = description
+ if outputFiles=="_auto":
+ outputFiles = self.shader+"x"
+ if outputFiles is None:
+ outputFiles = []
+ if isinstance(outputFiles, basestring):
+ outputFiles = [outputFiles]
+ self.outputFiles = outputFiles
+
+ self.stdoutFilter = None
+ self.stderrFilter = r"^\d\d/\d\d/\d\d\d\d \d\d:\d\d:\d\d"
+ self.outputFilter = r"AQSIS_V \d+.\d+.\d+"
+
+ # Add job to the global list if it's allowed.
+ if jobAllowed(self):
+ AllCmdJobs.append(self)
+
+
+######################################################################
+
# RenderJob
class RenderJob:
"""This class holds the input data required to render one RIB file.
@@ -350,6 +418,7 @@
# The absolute path to the working directory
self.workingdir = workingdir
self.rib = rib
+ self.name = rib
self.shaders = shaders
# outimagename: List of output image names
self.outimagename = outimagename
@@ -590,7 +659,7 @@
for sh in job.shaders:
if not os.path.isabs(sh):
sh = os.path.normpath(os.path.join(job.workingdir, sh))
- out,err = self.compileShader(sh)
+ retval,out,err = self.compileShader(sh)
shinfo.append((sh,out,err))
finally:
ShaderLock.release()
@@ -663,7 +732,7 @@
if self.compiledSLExt is not None and shader in self.compiledShaders:
compiledSL = os.path.splitext(shader)[0]+self.compiledSLExt
if not isNewer(shader, compiledSL):
- return "rendertester: %s was still up to date"%compiledSL,""
+ return 0,"rendertester: %s was still up to date"%compiledSL,""
self.compiledShaders[shader] = 1
@@ -675,7 +744,7 @@
retval,out,err = execute( self.shadercompiler + [shname], cwd=shaderdir, env=self.environ )
- return out,err
+ return retval,out,err
# shaderCompilerVersion
def shaderCompilerVersion(self):
@@ -703,11 +772,15 @@
class HTMLPage:
"""A simple wrapper around HTML pages."""
- def __init__(self, filename, title):
+ def __init__(self, filename, title, stylesheet=None):
self.fhandle = file(filename, "w")
- self.fhandle.write("""
-<html><head><title>%s</title></head>
-<body>\n"""%(title))
+ self.fhandle.write("<html><head><title>%s</title>"%title)
+ if stylesheet is not None:
+ self.fhandle.write('<style type="text/css">\n')
+ self.fhandle.write(stylesheet+"\n")
+ self.fhandle.write("</style>\n")
+ self.fhandle.write("</head>\n")
+ self.fhandle.write("<body>\n")
def __del__(self):
if self.fhandle!=None:
@@ -2077,7 +2150,7 @@
def __init__(self, jobs=AllJobs):
"""Constructor.
- jobs - Jobs that'll be included in the overview.
+ jobs - Jobs that will be included in the overview.
"""
Task.__init__(self)
self.jobs = self.convertJobList(jobs)
@@ -2241,7 +2314,295 @@
os.chdir(olddir)
return res
+######################################################################
+
+class TableRow:
+ """A helper class to create a row from a HTML table.
+ """
+ def __init__(self, cells=None):
+ # A list of dicts
+ self._columns = []
+ if cells is not None:
+ self.setText(0, cells)
+
+ def __setitem__(self, idx, value):
+ self.setText(idx, value)
+
+ def clear(self):
+ """Clear all rows.
+ """
+ self._columns = []
+
+ def getText(self, col):
+ """Return the cell contents of cell col.
+ """
+ self._assertCell(col)
+ return self._columns[col]["_data"]
+
+ def setText(self, col, txt):
+ """Set the cell contents of cell col.
+
+ txt can be a single string or a list of strings in which case
+ several cells starting at col will be filled.
+ """
+ if isinstance(txt, basestring):
+ txt = [txt]
+ else:
+ try:
+ len(txt)
+ except:
+ txt = [txt]
+ for i,t in enumerate(txt):
+ self.setAttr(col+i, "_data", t)
+
+ def setAttr(self, col, attr, val):
+ """Set a cell attribute.
+ """
+ self._assertCell(col)
+ self._columns[col][attr] = val
+
+ def toHtml(self):
+ """Return a string containing HTML code for the row.
+ """
+ res = "<tr>"
+ for attrs in self._columns:
+ data = attrs["_data"]
+ td = ["td"]
+ for attr,val in attrs.iteritems():
+ if attr!="_data":
+ td.append('%s="%s"'%(attr,val))
+ res += "<%s>%s</td>"%(" ".join(td), data)
+ res += "</tr>"
+ return res
+
+ def _assertCell(self, col):
+ """Make sure column col exists.
+ """
+ while col>=len(self._columns):
+ self._columns += [dict(_data="")]
+
+
+# RunCommands
+class RunCommands(Task):
+ """Runs the commands jobs that do not output images.
+ """
+ # This number is added to the output file name to make it unique when
+ # more than one RunComands object is given.
+ count = 1
+ def __init__(self, renderer, reference, jobs=AllCmdJobs, title="Commands"):
+ """Constructor.
+
+ renderer - The renderer whose shading compiler should be used
+ reference - The directory containing the reference files
+ jobs - Job list
+ title - A title string for the output page
+ """
+ Task.__init__(self)
+ self.renderer = renderer
+ self.reference = os.path.abspath(reference)
+ self.jobs = jobs
+ self.title = title
+
+ def run(self, htmldir):
+
+ htmlname = "commands_%s.html"%(self.count)
+ self.count += 1
+
+ print "Running command jobs..."
+
+ stylesheet = """
+td.header { background-color:#d0d0d0; font-weight:bold; }
+td.result_ok { background-color:#a0ffa0 }
+td.result_failure { background-color:#ff0000; font-weight:bold; color:#ffffa0; }
+.result_failure a:link { color:#ffff00; }
+.result_failure a:visited { color:#c0c000; }
+"""
+
+ html = HTMLPage(os.path.join(htmldir, htmlname), "Commands", stylesheet=stylesheet)
+ html.section(self.title)
+
+ html.write("<table border=1 cellspacing=1 cellpadding=2>")
+ row = TableRow(cells=["Test Name", "RetVal", "StdOut", "StdErr", "Shaders", "Description"])
+ for i in range(6):
+ row.setAttr(i, "class", "header")
+ html.write(row.toHtml())
+ numFailures = 0
+ for job in self.jobs:
+ row.clear()
+ row.setText(1, ["-", "-", "-", "-"])
+ # The compiled shader
+ for outFile in job.outputFiles:
+ if os.path.exists(outFile):
+ os.remove(outFile)
+ row[0] = job.name
+
+ retval,out,err = self.renderer.compileShader(job.shader)
+ # Check the return value
+ row.setAttr(1, "align", "center")
+ if job.returnCode is not None:
+ if retval==job.returnCode:
+ row[1] = retval
+ row.setAttr(1, "class", "result_ok")
+ else:
+ row[1] = "%s!=%s"%(retval, job.returnCode)
+ row.setAttr(1, "class", "result_failure")
+ # Check stdout output
+ self.setOutputCell(row, 2, htmldir, job, job.stdoutData, out, "stdout (ref)", "stdout", job.stdoutFilter)
+ # Check stderr output
+ self.setOutputCell(row, 3, htmldir, job, job.stderrData, err, "stderr (ref)", "stderr", job.stderrFilter)
+ # Check output file
+ if job.outputFiles!=[]:
+ row.setAttr(4, "class", "result_ok")
+ isFailure = False
+ txts = []
+ for outFile in job.outputFiles:
+ if os.path.exists(outFile):
+ f = open(outFile, "rt")
+ data = f.readlines()
+ f.close()
+ outBaseName = os.path.basename(outFile)
+ refName = os.path.join(self.reference, os.path.basename(outFile))
+ if os.path.exists(refName):
+ f = open(refName, "rt")
+ refLines = f.readlines()
+ f.close()
+ else:
+ refLines = ['Reference file "%s" is missing.'%refName]
+ ok,txt = self.compareText(htmldir, job, refLines, data, "%s (ref)"%outBaseName, outBaseName, job.outputFilter, True)
+ else:
+ ok = False
+ txt = '%s: missing'%os.path.basename(outFile)
+ txts.append(txt)
+ if not ok:
+ row.setAttr(4, "class", "result_failure")
+ isFailure = True
+ row[4] = ", ".join(txts)
+ if isFailure:
+ numFailures += 1
+ # Set the description (only the first sentence).
+ if job.description is not None:
+ desc = job.description
+ n = desc.find(".")
+ if n!=-1:
+ desc = desc[:n+1]
+ row[5] = desc
+ html.write(row.toHtml())
+ html.write("</table>")
+
+ if numFailures==0:
+ status = "no failures"
+ else:
+ status = "<font color=#ff0000><b>%s</b></font> failures"%numFailures
+ return (htmlname, status)
+
+ def setOutputCell(self, row, col, htmldir, job, refLines, lines, refDesc, desc, reFilter):
+ row.setAttr(col, "align", "center")
+ ok,txt = self.compareText(htmldir, job, refLines, lines, refDesc, desc, reFilter)
+ row[col] = txt
+ if ok:
+ if txt=="OK":
+ row.setAttr(col, "class", "result_ok")
+ else:
+ row.setAttr(col, "class", "result_failure")
+
+ def compareText(self, htmldir, job, refLines, lines, refDesc, desc, reFilter, prefix=False):
+ """Compare two texts and produce a diff html file if they are different.
+
+ htmldir is the directory where to write the diff file. job is the current
+ job (this is used for making the output name unique).
+ refLines and lines are lists of strings containing the reference text
+ and the actual text. refDesc and desc are the table headers describing
+ each text (e.g. file name).
+ reFilter is a string containing a regular expression that is used
+ to filter the texts (may be empty or None).
+
+ The return value is a tuple (Status, Text) where Status is a boolean
+ that indicates whether the test passed or failed. Text is a string
+ that has to be put into the resulting table (it is either OK or a link
+ to the diff page).
+ If prefix is True, the returned string will contain desc as a prefix
+ (to identify the output file).
+ """
+ if refLines is None:
+ return True, "-"
+
+ # If refLines/lines are just string, then split them up into lines
+ if isinstance(refLines, basestring):
+ if refLines=="":
+ refLines = []
+ else:
+ refLines = refLines.split("\n")
+ if isinstance(lines, basestring):
+ lines = lines.split("\n")
+
+ diffFile = self._compareText(htmldir, job, refLines, lines, refDesc, desc, reFilter)
+ if prefix:
+ prefixStr = "%s:"%desc
+ else:
+ prefixStr = ""
+ if diffFile is None:
+ return True, "%sOK"%prefixStr
+ else:
+ return False, '%s<a href="%s">Failure</a>'%(prefixStr,diffFile)
+
+ def _compareText(self, htmldir, job, refLines, lines, refDesc, desc, reFilter):
+ """Compare two texts and produce a diff html file if they are different.
+
+ htmldir is the directory where to write the diff file. job is the current
+ job (this is used for making the output name unique).
+ refLines and lines are lists of strings containing the reference text
+ and the actual text. refDesc and desc are the table headers describing
+ each text (e.g. file name).
+ reFilter is a string containing a regular expression that is used
+ to filter the texts (may be empty or None).
+ If the texts are identical, the method returns None and no file is
+ written, otherwise a diff file is written and the return value is
+ the (relative) name of the file.
+ """
+ # Filter the lines...
+ refLines = self.filterLines(refLines, reFilter)
+ lines = self.filterLines(lines, reFilter)
+ # Check if the texts are identical...
+ if lines==refLines:
+ return None
+ # Not identical, so create a diff html page...
+ htmlBaseName = os.path.splitext(os.path.basename(job.name))[0]
+ htmlBaseName = "%s_%s_%s.html"%(htmlBaseName, desc, id(job))
+ htmlName = os.path.join(htmldir, htmlBaseName)
+ hdiff = difflib.HtmlDiff()
+ s = hdiff.make_file(refLines, lines, refDesc, desc)
+ f = open(htmlName, "wt")
+ f.write(s)
+ f.close()
+ return htmlBaseName
+
+ def filterLines(self, lines, reFilter):
+ """Filter lines using a regular expression.
+
+ lines is a list containing strings. reFilter is a string containing
+ a regular expression. Each string in lines is searched for the expression
+ and the matching text is replaced by stars.
+ The return value is a list containing the filtered lines.
+
+ """
+ if reFilter is None:
+ reFilter = ""
+ rexp = re.compile(reFilter)
+ res = []
+ for line in lines:
+ line = line.rstrip()
+ m = rexp.search(line)
+ if m is not None:
+ s = m.start()
+ e = m.end()
+ # Replace the matched text with stars...
+ line = line[:s]+(e-s)*"*"+line[e:]
+ res.append(line)
+ return res
+
+
+
######################################################################
# preparePath
@@ -2398,7 +2759,7 @@
include_cfg(tasks_cfg)
print "%d renderer definitions"%len(Renderers)
- print "%d jobs"%len(AllJobs)
+ print "%d render jobs / %d command jobs"%(len(AllJobs), len(AllCmdJobs))
print "%d tasks"%len(Tasks)
# Override the number of processes if specified on the command line
Modified: trunk/testing/regression/tasks.cfg
===================================================================
--- trunk/testing/regression/tasks.cfg 2008-08-24 03:44:03 UTC (rev 2402)
+++ trunk/testing/regression/tasks.cfg 2008-08-25 08:25:26 UTC (rev 2403)
@@ -112,5 +112,9 @@
ricalls = RiCalls( jobs = AllJobs )
+rt_aqsl = RunCommands( renderer = aqsis,
+ jobs = AllCmdJobs,
+ reference = "referenceShaders",
+ title = "Shader Compiler Tests")
include_cfg("custom_tasks.cfg")
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|