## cgkit-commits

 [cgkit-commits] SF.net SVN: cgkit:[252] cgkit/trunk From: - 2008-08-25 15:09:02 Revision: 252 http://cgkit.svn.sourceforge.net/cgkit/?rev=252&view=rev Author: mbaas Date: 2008-08-25 15:08:52 +0000 (Mon, 25 Aug 2008) Log Message: ----------- Replaced the broken toEuler*() methods with a more robust version (taken from the comp.graphics.algorithms FAQ wiki). Modified Paths: -------------- cgkit/trunk/cgkit/light/cgtypes/mat3.py cgkit/trunk/changelog.txt cgkit/trunk/supportlib/include/mat3.h cgkit/trunk/supportlib/include/vec3.h cgkit/trunk/unittests/test_mat3.py cgkit/trunk/unittests/test_mat3_light.py Modified: cgkit/trunk/cgkit/light/cgtypes/mat3.py =================================================================== --- cgkit/trunk/cgkit/light/cgtypes/mat3.py 2008-08-25 15:05:45 UTC (rev 251) +++ cgkit/trunk/cgkit/light/cgtypes/mat3.py 2008-08-25 15:08:52 UTC (rev 252) @@ -595,140 +595,32 @@ def toEulerYXZ(self): """Return the Euler angles of a rotation matrix.""" - global _epsilon - - r1 = self.getRow(0) - r2 = self.getRow(1) - r3 = self.getRow(2) - - B = -r2.z - - x = math.asin(B) - - A = math.cos(x) - - if (A>_epsilon): - y = math.acos(r3.z/A) - z = math.acos(r2.y/A) - else: - z = 0.0 - y = math.acos(r1.x) - + y,x,z = self._getRotation(2, True, True, True) return (x,y,z) def toEulerZXY(self): """Return the Euler angles of a rotation matrix.""" - global _epsilon - - r1 = self.getRow(0) - r2 = self.getRow(1) - r3 = self.getRow(2) - - B = r3.y - - x = math.asin(B) - - A = math.cos(x) - - if (A>_epsilon): - y = math.acos(r3.z/A) - z = math.acos(r2.y/A) - else: - z = 0.0 - y = math.acos(r1.x) - + z,x,y = self._getRotation(1, False, True, True) return (x,y,z) def toEulerZYX(self): """Return the Euler angles of a rotation matrix.""" - global _epsilon - - r1 = self.getRow(0) - r2 = self.getRow(1) - r3 = self.getRow(2) - - D = -r3.x - - y = math.asin(D) - - C = math.cos(y) - - if (C>_epsilon): - x = math.acos(r3.z/C) - z = math.acos(r1.x/C) - else: - z = 0.0 - x = math.acos(-r2.y) - + z,y,x = self._getRotation(0, True, True, True) return (x,y,z) def toEulerYZX(self): """Return the Euler angles of a rotation matrix.""" - global _epsilon - - r1 = self.getRow(0) - r2 = self.getRow(1) - r3 = self.getRow(2) - - F = r2.x - - z = math.asin(F) - - E = math.cos(z) - - if (E>_epsilon): - x = math.acos(r2.y/E) - y = math.acos(r1.x/E) - else: - y = 0.0 - x = math.asin(r3.y) - + y,z,x = self._getRotation(0, False, True, True) return (x,y,z) def toEulerXZY(self): """Return the Euler angles of a rotation matrix.""" - global _epsilon - - r1 = self.getRow(0) - r2 = self.getRow(1) - r3 = self.getRow(2) - - F = -r1.y - - z = math.asin(F) - - E = math.cos(z) - - if (E>_epsilon): - x = math.acos(r2.y/E) - y = math.acos(r1.x/E) - else: - y = 0.0 - x = math.acos(r3.z) - + x,z,y = self._getRotation(1, True, True, True) return (x,y,z) def toEulerXYZ(self): """Return the Euler angles of a rotation matrix.""" - global _epsilon - - r1 = self.getRow(0) - r2 = self.getRow(1) - r3 = self.getRow(2) - - D = r1.z - - y = math.asin(D) - - C = math.cos(y) - - if (C>_epsilon): - x = math.acos(r3.z/C) - z = math.acos(r1.x/C) - else: - z = 0.0 - x = math.acos(r2.y) - + x,y,z = self._getRotation(2, False, True, True) return (x,y,z) def fromToRotation(_from, to): @@ -809,6 +701,94 @@ fromToRotation = staticmethod(fromToRotation) + def _getRotation(self, i, neg, alt, rev): + """Get Euler angles in any of the 24 different conventions. + + The first four argument select a particular convention. The last three + output arguments receive the angles. The order of the angles depends + on the convention. + + See http://www.cgafaq.info/wiki/Euler_angles_from_matrix for the + algorithm used. + + i: The index of the first axis (global rotations, s) or last axis (local rotations, r). 0=XZX, 1=YXY, 2=ZYZ + neg: If true, the convention contains an odd permutation of the convention defined by i alone (i.e. the middle axis is replaced. For example, XZX -> XYX) + alt: If true, the first and last axes are different. Local rotations: The first axis changes. For example, XZX -> YZX + rev: If true, the first and last angle are exchanged. This toggles between global/local rotations. In all the concrete getRotation*() functions this is always true because all the functions assume local rotations. + """ + v = [self[0,i], self[1,i], self[2,i]] + + j,k,h = _eulerIndices(i, neg, alt) + + a = v[h] + b = v[k] + c,s,r = _eulerGivens(a, b) + v[h] = r + s1 = c*self[k,j] - s*self[h,j] + c1 = c*self[k,k] - s*self[h,k] + r1 = math.atan2(s1, c1) + r2 = math.atan2(v[j], v[i]) + r3 = math.atan2(s, c) + if alt: + r3 = -r3 + if neg: + r1 = -r1 + r2 = -r2 + r3 = -r3 + if rev: + tmp = r1 + r1 = r3 + r3 = tmp + return r1,r2,r3 + +def _eulerIndices(i, neg, alt): + """Helper function for _getRotation().""" + next = [1, 2, 0, 1] + j = next[i+int(neg)] + k = 3-i-j + h = next[k+(1^int(neg)^int(alt))] + return j,k,h + +def _eulerGivens(a, b): + """Helper function for _getRotation().""" + global _epsilon + + absa = abs(a) + absb = abs(b) + # b=0? + if absb<=_epsilon: + if a>=0: + c = 1.0 + else: + c = -1.0 + return (c, 0.0, absa) + # a=0? + elif absa<=_epsilon: + if b>=0: + s = 1.0 + else: + s = -1.0 + return (0.0, s, absb) + # General case + else: + if absb>absa: + t = a/b + u = math.sqrt(1.0+t*t) + if b<0: + u = -u + s = 1.0/u + c = s*t + r = b*u + else: + t = b/a + u = math.sqrt(1.0+t*t) + if (a<0): + u = -u + c = 1.0/u + s = c*t + r = a*u + return c,s,r + ###################################################################### if __name__=="__main__": Modified: cgkit/trunk/changelog.txt =================================================================== --- cgkit/trunk/changelog.txt 2008-08-25 15:05:45 UTC (rev 251) +++ cgkit/trunk/changelog.txt 2008-08-25 15:08:52 UTC (rev 252) @@ -5,6 +5,15 @@ Bug fixes/enhancements: +- mat3: The toEuler*() methods were broken. Depending on the angles, they + could return invalid values. +- sl/noise: When only cgkit light was installed, the sl module couldn't be + imported because it imports noise which has no pure Python equivalent yet. + There is a dummy noise module now in the light version which allows importing + the module. An error is only triggered when one of the functions gets called. +- rply: The sources can now also be compiled using a C++ compiler. Thanks + to Chris Foster for the patch! +- ribexport: Added RenderMan support for the torus geometry. - ri/cri: Passing unicode strings in a parameter list has caused an exception. ---------------------------------------------------------------------- Modified: cgkit/trunk/supportlib/include/mat3.h =================================================================== --- cgkit/trunk/supportlib/include/mat3.h 2008-08-25 15:05:45 UTC (rev 251) +++ cgkit/trunk/supportlib/include/mat3.h 2008-08-25 15:08:52 UTC (rev 252) @@ -119,6 +119,7 @@ mat3(const mat3& A); T& at(short i, short j); + const T& at(short i, short j) const; // set_ and get_ methods mat3& setIdentity(); @@ -219,6 +220,10 @@ vec3 r2; vec3 r3; + void _getRotation(int i, bool neg, bool alt, bool rev, T& r1, T& r2, T& r3) const; + void _eulerIndices(int i, bool neg, bool alt, int &j, int &k, int&h) const; + void _eulerGivens(T a, T b, T& c, T& s, T& v) const; + // Constructor with three vectors: Init the rows with those vectors. // This constructor is private because its semantics depends // on the actual implementation (that is, the three vectors are @@ -380,7 +385,30 @@ } } +/** + Return element [i,j]. + The indices range from 0 to 2. Other values will cause an EIndexError + exception to be thrown. + + @param i Row + @param j Column + @return Reference to element [i,j] + @exception EIndexError + */ +template +inline const T& mat3::at(short i, short j) const +{ + switch(i) + { + case 0: return r1[j]; + case 1: return r2[j]; + case 2: return r3[j]; + default: throw EIndexError(); + } +} + + /*----------------------------setIdentity------------------------------*//** Set identity matrix. @@ -1584,34 +1612,136 @@ } /** - Get Euler angles (order: yxz). + Helper function for the _getRotation() function. + */ +template +void mat3::_eulerIndices(int i, bool neg, bool alt, int& j, int& k, int& h) const +{ + int next[4] = {1, 2, 0, 1}; + j = next[i+int(neg)]; + k = 3-i-j; + h = next[k+(1^int(neg)^int(alt))]; +} - \param[out] x Angle around x (radians) - \param[out] y Angle around y (radians) - \param[out] z Angle around z (radians) +/** + Helper function for the _getRotation() function. */ template -void mat3::getRotationYXZ(T& x, T& y, T& z) const +void mat3::_eulerGivens(T a, T b, T& c, T& s, T& r) const { - T B = -r2.z; + T absa = xabs(a); + T absb = xabs(b); + // b=0? + if (absb<=vec3::epsilon) + { + if (a>=0) + c = 1.0; + else + c = -1.0; + s = 0.0; + r = absa; + } + // a=0? + else if (absa<=vec3::epsilon) + { + c = 0.0; + if (b>=0) + s = 1.0; + else + s = -1.0; + r = absb; + } + // General case + else + { + if (absb>absa) + { + T t = a/b; + T u = sqrt(1.0+t*t); + if (b<0) + u = -u; + s = 1.0/u; + c = s*t; + r = b*u; + } + else + { + T t = b/a; + T u = sqrt(1.0+t*t); + if (a<0) + u = -u; + c = 1.0/u; + s = c*t; + r = a*u; + } + } +} - x = asin(B); +/** + Get Euler angles in any of the 24 different conventions. - T A = cos(x); + The first four argument select a particular convention. The last three + output arguments receive the angles. The order of the angles depends + on the convention. - if (A>vec3::epsilon) + See http://www.cgafaq.info/wiki/Euler_angles_from_matrix for the + algorithm used. + + \param i The index of the first axis (global rotations, s) or last axis (local rotations, r). 0=XZX, 1=YXY, 2=ZYZ + \param neg If true, the convention contains an odd permutation of the convention defined by i alone (i.e. the middle axis is replaced. For example, XZX -> XYX) + \param alt If true, the first and last axes are different. Local rotations: The first axis changes. For example, XZX -> YZX + \param rev If true, the first and last angle are exchanged. This toggles between global/local rotations. In all the concrete getRotation*() functions this is always true because all the functions assume local rotations. + \param[out] r1 Angle around first axis (radians) + \param[out] r2 Angle around second axis (radians) + \param[out] r3 Angle around third axis (radians) + */ +template +void mat3::_getRotation(int i, bool neg, bool alt, bool rev, T& r1, T& r2, T& r3) const +{ + int j, k, h; + T v[3] = {at(0,i), at(1,i), at(2,i)}; + + _eulerIndices(i, neg, alt, j, k, h); + + T a = v[h]; + T b = v[k]; + T c, s; + _eulerGivens(a, b, c, s, v[h]); + T s1 = c*at(k,j) - s*at(h,j); + T c1 = c*at(k,k) - s*at(h,k); + r1 = atan2(s1, c1); + r2 = atan2(v[j], v[i]); + r3 = atan2(s, c); + if (alt) + r3 = -r3; + if (neg) { - y = acos(r3.z/A); - z = acos(r2.y/A); + r1 = -r1; + r2 = -r2; + r3 = -r3; } - else + if (rev) { - z = 0; - y = acos(r1.x); - } + T tmp = r1; + r1 = r3; + r3 = tmp; + } } /** + Get Euler angles (order: yxz). + + \param[out] x Angle around x (radians) + \param[out] y Angle around y (radians) + \param[out] z Angle around z (radians) + */ +template +void mat3::getRotationYXZ(T& x, T& y, T& z) const +{ + _getRotation(2, 1, 1, 1, y, x, z); +} + +/** Get Euler angles (order: zxy). \param[out] x Angle around x (radians) @@ -1621,22 +1751,7 @@ template void mat3::getRotationZXY(T& x, T& y, T& z) const { - T B = r3.y; - - x = asin(B); - - T A = cos(x); - - if (A>vec3::epsilon) - { - y = acos(r3.z/A); - z = acos(r2.y/A); - } - else - { - z = 0; - y = acos(r1.x); - } + _getRotation(1, 0, 1, 1, z, x, y); } /** @@ -1649,22 +1764,7 @@ template void mat3::getRotationZYX(T& x, T& y, T& z) const { - T D = -r3.x; - - y = asin(D); - - T C = cos(y); - - if (C>vec3::epsilon) - { - x = acos(r3.z/C); - z = acos(r1.x/C); - } - else - { - z = 0; - x = acos(-r2.y); - } + _getRotation(0, 1, 1, 1, z, y, x); } /** @@ -1677,22 +1777,7 @@ template void mat3::getRotationYZX(T& x, T& y, T& z) const { - T F = r2.x; - - z = asin(F); - - T E = cos(z); - - if (E>vec3::epsilon) - { - x = acos(r2.y/E); - y = acos(r1.x/E); - } - else - { - y = 0; - x = asin(r3.y); - } + _getRotation(0, 0, 1, 1, y, z, x); } /** @@ -1705,22 +1790,7 @@ template void mat3::getRotationXZY(T& x, T& y, T& z) const { - T F = -r1.y; - - z = asin(F); - - T E = cos(z); - - if (E>vec3::epsilon) - { - x = acos(r2.y/E); - y = acos(r1.x/E); - } - else - { - y = 0; - x = acos(r3.z); - } + _getRotation(1, 1, 1, 1, x, z, y); } /** @@ -1733,22 +1803,7 @@ template void mat3::getRotationXYZ(T& x, T& y, T& z) const { - T D = r1.z; - - y = asin(D); - - T C = cos(y); - - if (C>vec3::epsilon) - { - x = acos(r3.z/C); - z = acos(r1.x/C); - } - else - { - z = 0; - x = acos(r2.y); - } + _getRotation(2, 0, 1, 1, x, y, z); } /** Modified: cgkit/trunk/supportlib/include/vec3.h =================================================================== --- cgkit/trunk/supportlib/include/vec3.h 2008-08-25 15:05:45 UTC (rev 251) +++ cgkit/trunk/supportlib/include/vec3.h 2008-08-25 15:08:52 UTC (rev 252) @@ -170,6 +170,7 @@ void get_polar(T& r, T& theta, T& phi) const; T& operator[](int); + const T& operator[](int) const; vec3& operator+=(const vec3& v); vec3& operator-=(const vec3& v); @@ -415,6 +416,27 @@ } } +/*-----------------------------operator[]-------------------------------*//** + Index operator. + + Return component i. The index is checked. + + @param index Component index (0,1 or 2). + @return Component i. + @exception EIndexError Index is out of range. +*//*------------------------------------------------------------------------*/ +template +const T& vec3::operator[](int index) const +{ + switch (index) + { + case 0: return x; + case 1: return y; + case 2: return z; + default: throw EIndexError(); + } +} + /*-----------------------------operator+=-------------------------------*//** vector += vector. Modified: cgkit/trunk/unittests/test_mat3.py =================================================================== --- cgkit/trunk/unittests/test_mat3.py 2008-08-25 15:05:45 UTC (rev 251) +++ cgkit/trunk/unittests/test_mat3.py 2008-08-25 15:08:52 UTC (rev 252) @@ -1,10 +1,10 @@ # Test the mat3 class import unittest +#from cgkit.light.cgtypes import * from cgkit.cgtypes import * -#from cgkit.light.cgtypes import * import math, os, pickle, cPickle, sys -from cgkit.sl import radians +from cgkit.sl import degrees, radians class TestMat3(unittest.TestCase): @@ -511,26 +511,34 @@ matrix that is composed by 3 individual rotations. """ - angle = {"X":radians(20), "Y":radians(30), "Z":radians(40)} + angles = [{"X":radians(20), "Y":radians(30), "Z":radians(40)}, + {"X":radians(0), "Y":radians(0), "Z":radians(0)}, + {"X":radians(350), "Y":radians(0), "Z":radians(0)}, + {"X":radians(0), "Y":radians(350), "Z":radians(0)}, + {"X":radians(0), "Y":radians(0), "Z":radians(350)},] axis = {"X":vec3(1,0,0), "Y":vec3(0,1,0), "Z":vec3(0,0,1)} for order in ["XYZ", "YZX", "ZXY", "XZY", "YXZ", "ZYX"]: - R1 = mat3.rotation(angle[order[0]], axis[order[0]]) - R2 = mat3.rotation(angle[order[1]], axis[order[1]]) - R3 = mat3.rotation(angle[order[2]], axis[order[2]]) - # Each rotation is about the *global* axis, so these rotations - # have to be applied just in the opposite order than mentioned - # in the fromEuler*() method name. - C = R1*R2*R3 - exec 'E = mat3.fromEuler%s(angle["X"], angle["Y"], angle["Z"])'%order - self.assertEqual(E, C) + for angle in angles: + R1 = mat3.rotation(angle[order[0]], axis[order[0]]) + R2 = mat3.rotation(angle[order[1]], axis[order[1]]) + R3 = mat3.rotation(angle[order[2]], axis[order[2]]) + # Each rotation is about the *global* axis, so these rotations + # have to be applied just in the opposite order than mentioned + # in the fromEuler*() method name. + C = R1*R2*R3 + exec 'E = mat3.fromEuler%s(angle["X"], angle["Y"], angle["Z"])'%order + self.assertEqual(E, C) + + exec 'x,y,z = E.toEuler%s()'%order + exec 'E2 = mat3.fromEuler%s(x, y, z)'%order + if E2!=E: +# print E +# print E2 + msg = "The matrix E2 generated from the toEuler() angles doesn't match the original matrix E.\n" + msg += "Original angles: (%s, %s, %s), toEuler angles: (%s, %s, %s)"%(degrees(angle["X"]),degrees(angle["Y"]),degrees(angle["Z"]),degrees(x),degrees(y),degrees(z)) + self.fail(msg) - exec 'x,y,z = E.toEuler%s()'%order - self.assertAlmostEqual(x, angle["X"], 8) - self.assertAlmostEqual(y, angle["Y"], 8) - self.assertAlmostEqual(z, angle["Z"], 8) - - ###################################################################### def testFromToRotation(self): R = mat3.fromToRotation((1,0,0), (0,1,0)) Modified: cgkit/trunk/unittests/test_mat3_light.py =================================================================== --- cgkit/trunk/unittests/test_mat3_light.py 2008-08-25 15:05:45 UTC (rev 251) +++ cgkit/trunk/unittests/test_mat3_light.py 2008-08-25 15:08:52 UTC (rev 252) @@ -1,10 +1,9 @@ # Test the mat3 class import unittest -#from cgkit.cgtypes import * from cgkit.light.cgtypes import * import math, os, pickle, cPickle, sys -from cgkit.sl import radians +from cgkit.sl import degrees, radians class TestMat3(unittest.TestCase): @@ -511,26 +510,34 @@ matrix that is composed by 3 individual rotations. """ - angle = {"X":radians(20), "Y":radians(30), "Z":radians(40)} + angles = [{"X":radians(20), "Y":radians(30), "Z":radians(40)}, + {"X":radians(0), "Y":radians(0), "Z":radians(0)}, + {"X":radians(350), "Y":radians(0), "Z":radians(0)}, + {"X":radians(0), "Y":radians(350), "Z":radians(0)}, + {"X":radians(0), "Y":radians(0), "Z":radians(350)},] axis = {"X":vec3(1,0,0), "Y":vec3(0,1,0), "Z":vec3(0,0,1)} for order in ["XYZ", "YZX", "ZXY", "XZY", "YXZ", "ZYX"]: - R1 = mat3.rotation(angle[order[0]], axis[order[0]]) - R2 = mat3.rotation(angle[order[1]], axis[order[1]]) - R3 = mat3.rotation(angle[order[2]], axis[order[2]]) - # Each rotation is about the *global* axis, so these rotations - # have to be applied just in the opposite order than mentioned - # in the fromEuler*() method name. - C = R1*R2*R3 - exec 'E = mat3.fromEuler%s(angle["X"], angle["Y"], angle["Z"])'%order - self.assertEqual(E, C) + for angle in angles: + R1 = mat3.rotation(angle[order[0]], axis[order[0]]) + R2 = mat3.rotation(angle[order[1]], axis[order[1]]) + R3 = mat3.rotation(angle[order[2]], axis[order[2]]) + # Each rotation is about the *global* axis, so these rotations + # have to be applied just in the opposite order than mentioned + # in the fromEuler*() method name. + C = R1*R2*R3 + exec 'E = mat3.fromEuler%s(angle["X"], angle["Y"], angle["Z"])'%order + self.assertEqual(E, C) + + exec 'x,y,z = E.toEuler%s()'%order + exec 'E2 = mat3.fromEuler%s(x, y, z)'%order + if E2!=E: +# print E +# print E2 + msg = "The matrix E2 generated from the toEuler() angles doesn't match the original matrix E.\n" + msg += "Original angles: (%s, %s, %s), toEuler angles: (%s, %s, %s)"%(degrees(angle["X"]),degrees(angle["Y"]),degrees(angle["Z"]),degrees(x),degrees(y),degrees(z)) + self.fail(msg) - exec 'x,y,z = E.toEuler%s()'%order - self.assertAlmostEqual(x, angle["X"], 8) - self.assertAlmostEqual(y, angle["Y"], 8) - self.assertAlmostEqual(z, angle["Z"], 8) - - ###################################################################### def testFromToRotation(self): R = mat3.fromToRotation((1,0,0), (0,1,0)) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. 
 [cgkit-commits] SF.net SVN: cgkit:[253] cgkit/trunk From: - 2008-09-14 09:07:08 Revision: 253 http://cgkit.svn.sourceforge.net/cgkit/?rev=253&view=rev Author: mbaas Date: 2008-09-14 09:07:04 +0000 (Sun, 14 Sep 2008) Log Message: ----------- The OBJMaterial was using a filter width of 0.1 which doesn't really make sense. Set it to 1.0 for now. Modified Paths: -------------- cgkit/trunk/cgkit/objmaterial.py cgkit/trunk/changelog.txt Modified: cgkit/trunk/cgkit/objmaterial.py =================================================================== --- cgkit/trunk/cgkit/objmaterial.py 2008-08-25 15:08:52 UTC (rev 252) +++ cgkit/trunk/cgkit/objmaterial.py 2008-09-14 09:07:04 UTC (rev 253) @@ -241,15 +241,15 @@ """Returns a list of RenderPass objects.""" texdefs = [] if self.map_Ka!=None: - texdefs.append((self.map_Ka.filename, "periodic", "periodic", "gaussian", 0.01, 0.01, {})) + texdefs.append((self.map_Ka.filename, "periodic", "periodic", "gaussian", 1.0, 1.0, {})) if self.map_Kd!=None: - texdefs.append((self.map_Kd.filename, "periodic", "periodic", "gaussian", 0.01, 0.01, {})) + texdefs.append((self.map_Kd.filename, "periodic", "periodic", "gaussian", 1.0, 1.0, {})) if self.map_Ks!=None: - texdefs.append((self.map_Ks.filename, "periodic", "periodic", "gaussian", 0.01, 0.01, {})) + texdefs.append((self.map_Ks.filename, "periodic", "periodic", "gaussian", 1.0, 1.0, {})) if self.map_d!=None: - texdefs.append((self.map_d.filename, "periodic", "periodic", "gaussian", 0.01, 0.01, {})) + texdefs.append((self.map_d.filename, "periodic", "periodic", "gaussian", 1.0, 1.0, {})) if self.map_Bump!=None: - texdefs.append((self.map_Bump.filename, "periodic", "periodic", "gaussian", 0.01, 0.01, {})) + texdefs.append((self.map_Bump.filename, "periodic", "periodic", "gaussian", 1.0, 1.0, {})) if texdefs!=[]: return [ribexport.TexPass(maps=texdefs)] Modified: cgkit/trunk/changelog.txt =================================================================== --- cgkit/trunk/changelog.txt 2008-08-25 15:08:52 UTC (rev 252) +++ cgkit/trunk/changelog.txt 2008-09-14 09:07:04 UTC (rev 253) @@ -5,6 +5,9 @@ Bug fixes/enhancements: +- ObjMaterial: Texture maps were rendered with a filter width of 0.01 which + doesn't make much sense. For now, I set it to 1.0, but this should probably + be customizable. - mat3: The toEuler*() methods were broken. Depending on the angles, they could return invalid values. - sl/noise: When only cgkit light was installed, the sl module couldn't be This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. 
 [cgkit-commits] SF.net SVN: cgkit:[254] cgkit/trunk From: - 2008-09-14 09:24:24 Revision: 254 http://cgkit.svn.sourceforge.net/cgkit/?rev=254&view=rev Author: mbaas Date: 2008-09-14 09:24:20 +0000 (Sun, 14 Sep 2008) Log Message: ----------- Initial version of the sloargs module to parse compiled shaders. Modified Paths: -------------- cgkit/trunk/cgkit/slparams.py cgkit/trunk/changelog.txt Added Paths: ----------- cgkit/trunk/cgkit/sloargs.py Added: cgkit/trunk/cgkit/sloargs.py =================================================================== --- cgkit/trunk/cgkit/sloargs.py (rev 0) +++ cgkit/trunk/cgkit/sloargs.py 2008-09-14 09:24:20 UTC (rev 254) @@ -0,0 +1,446 @@ +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is the Python Computer Graphics Kit. +# +# The Initial Developer of the Original Code is Matthias Baas. +# Portions created by the Initial Developer are Copyright (C) 2008 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +import sys, os.path +import ctypes +import ctypes.util + +class _POINT(ctypes.Structure): + _fields_ = [("xval", ctypes.c_float), + ("yval", ctypes.c_float), + ("zval", ctypes.c_float)] + +class _DEFAULTVAL(ctypes.Union): + _fields_ = [("pointval", ctypes.POINTER(_POINT)), + ("scalarval", ctypes.POINTER(ctypes.c_float)), + ("stringval", ctypes.c_char_p), + ("matrixval", ctypes.POINTER(16*ctypes.c_float)) + ] + +# PRMan 14 +class _VISSYMDEF_prman(ctypes.Structure): + _fields_ = [("name", ctypes.c_char_p), + ("type", ctypes.c_int), + ("storage", ctypes.c_int), + ("detail", ctypes.c_int), + ("spacename", ctypes.c_char_p), + ("default", _DEFAULTVAL), + ("defaultval", _POINT), + ("valisvalid", ctypes.c_uint), + ("arraylen", ctypes.c_int), + ] + +# 3Delight 7.0 +class _VISSYMDEF_3delight(ctypes.Structure): + _fields_ = [("name", ctypes.c_char_p), + ("type", ctypes.c_int), + ("storage", ctypes.c_int), + ("detail", ctypes.c_int), + ("spacename", ctypes.c_char_p), + ("default", _DEFAULTVAL), + ("valisvalid", ctypes.c_uint), + ("arraylen", ctypes.c_int), + ] + +# Aqsis +class _VISSYMDEF_aqsis(ctypes.Structure): + _fields_ = [("name", ctypes.c_char_p), + ("type", ctypes.c_int), + ("storage", ctypes.c_int), + ("detail", ctypes.c_int), + ("spacename", ctypes.c_char_p), + ("arraylen", ctypes.c_int), + ("default", _DEFAULTVAL), + ] + + +class _SloArgs: + """Provides functionality to read shader parameters using the C sloargs interface. + + Because of differences between renderers, the sloargs interface is not + exposed directly. Instead, all information about a shader is returned + by the getShaderInfo() method. + + This is the base class for the renderer-specific versions. + """ + + def __init__(self, libName, VISSYMDEF): + """Constructor. + + libName is the name of the library that implements the sloargs interface. + The name can either be just the base lib name or an absolute file name. + VISSYMDEF is a ctype Structure object that describes the VISSYMDEF + struct for this particular renderer. + """ + # Try to figure out the location of the lib if the name is not an absolute path... + if not os.path.isabs(libName): + p = ctypes.util.find_library(libName) + if p is not None: + libName = p + + self._VISSYMDEF = VISSYMDEF + + # Load the library... + try: + self._sloargs = self._loadLibrary(libName) + except: + raise ValueError('Failed to load library "%s": %s'%(libName, sys.exc_info()[1])) + try: + self._declareFunctions(self._sloargs) + except: + raise ValueError('Library "%s" does not implement the sloargs interface: %s'%(libName, sys.exc_info()[1])) + + def _loadLibrary(self, libName): + """Load the library providing the sloargs interface. + + libName is the (resolved) file name of the library to load. + The return value is the ctypes library that must contain the + sloargs functions as defined by PRMan. + """ + sloargs = ctypes.cdll.LoadLibrary(libName) + return sloargs + + def _declareFunctions(self, sloargs): + """Declare the arguments and return types of the library functions. + + sloargs is the ctypes library as returned by LoadLibrary(). + This method can be overridden by derived classes to add renderer-specific + functions. The derived class must call the inherited function. + """ + VISSYMDEF = self._VISSYMDEF + sloargs.Slo_SetPath.argtypes = [ctypes.c_char_p] + sloargs.Slo_SetPath.restype = ctypes.c_int + sloargs.Slo_SetShader.argtypes = [ctypes.c_char_p] + sloargs.Slo_SetShader.restype = ctypes.c_int + sloargs.Slo_GetName.argtypes = [] + sloargs.Slo_GetName.restype = ctypes.c_char_p + sloargs.Slo_GetType.argtypes = [] + sloargs.Slo_GetType.restype = ctypes.c_int + sloargs.Slo_GetNArgs.argtypes = [] + sloargs.Slo_GetNArgs.restype = ctypes.c_int + sloargs.Slo_GetArgById.argtypes = [ctypes.c_int] + sloargs.Slo_GetArgById.restype = ctypes.POINTER(VISSYMDEF) + sloargs.Slo_GetArgByName.argtypes = [ctypes.c_char_p] + sloargs.Slo_GetArgByName.restype = ctypes.POINTER(VISSYMDEF) + sloargs.Slo_GetArrayArgElement.argtypes = [ctypes.POINTER(VISSYMDEF), ctypes.c_int] + sloargs.Slo_GetArrayArgElement.restype = ctypes.POINTER(VISSYMDEF) + sloargs.Slo_EndShader.argtypes = [] + sloargs.Slo_EndShader.restype = None + sloargs.Slo_TypetoStr.argtypes = [ctypes.c_int] + sloargs.Slo_TypetoStr.restype = ctypes.c_char_p + sloargs.Slo_StortoStr.argtypes = [ctypes.c_int] + sloargs.Slo_StortoStr.restype = ctypes.c_char_p + sloargs.Slo_DetailtoStr.argtypes = [ctypes.c_int] + sloargs.Slo_DetailtoStr.restype = ctypes.c_char_p + + def getShaderInfo(self, shader): + """Read the shader parameters from a given shader. + """ + sloargs = self._sloargs + if sloargs.Slo_SetShader(shader)!=0: + raise IOError('Failed to open shader "%s"'%shader) + + shaderName = sloargs.Slo_GetName() + shaderType = sloargs.Slo_TypetoStr(sloargs.Slo_GetType()) + params = [] + numParams = sloargs.Slo_GetNArgs() + for i in range(1, numParams+1): + res = sloargs.Slo_GetArgById(i) + symdef = res.contents + + if sloargs.Slo_StortoStr(symdef.storage).startswith("output"): + output = "output" + else: + output = "" + storage = sloargs.Slo_DetailtoStr(symdef.detail) + paramType = sloargs.Slo_TypetoStr(symdef.type) + arrLen = symdef.arraylen + if arrLen==0: + arrLen = None + name = symdef.name + space = self._getSpace(symdef) + if space=="": + space = None + defaultVal = self._getDefaultVal(symdef) + params.append((output,storage,paramType,arrLen,name,space,defaultVal)) + + sloargs.Slo_EndShader() + return [(shaderType, shaderName, params)] + + def _getSpace(self, symdef): + if symdef.arraylen>0: + res = [] + for i in range(symdef.arraylen): + psd = self._sloargs.Slo_GetArrayArgElement(symdef, i) + space = self._getSpace(psd.contents) + if space=="": + space = None + res.append(space) + # Did we only get a list of None objects? + # Then just return None (as this has probably been a type + # that has no spaces associated with it) + if res==len(res)*[None]: + res = None + return res + else: + return symdef.spacename + + def _getDefaultVal(self, symdef): + if symdef.arraylen>0: + res = [] + for i in range(symdef.arraylen): + psd = self._sloargs.Slo_GetArrayArgElement(symdef, i) + res.append(self._getDefaultVal(psd.contents)) + return res + + typ = self._sloargs.Slo_TypetoStr(symdef.type) + # Scalar? + if typ=="float": + return symdef.default.scalarval.contents.value + # Color, Point, Normal, Vector? + elif typ in ["color", "point", "vector", "normal"]: + v = symdef.default.pointval.contents + return (v.xval, v.yval, v.zval) + # String? + elif typ=="string": + return symdef.default.stringval + # Matrix? + elif typ=="matrix": + return tuple(symdef.default.matrixval.contents) + + return None + + +class _SloArgs_PRMan(_SloArgs): + def __init__(self): + _SloArgs.__init__(self, libName="prman", VISSYMDEF=_VISSYMDEF_prman) + + +class _SloArgs_3Delight(_SloArgs): + def __init__(self): + _SloArgs.__init__(self, libName="3delight", VISSYMDEF=_VISSYMDEF_3delight) + + def _declareFunctions(self, sloargs): + _SloArgs._declareFunctions(self, sloargs) + # 3Delight-specific functions + sloargs.Slo_GetNAnnotations.argtypes = [] + sloargs.Slo_GetNAnnotations.restype = ctypes.c_int + sloargs.Slo_GetAnnotationKeyById.argtypes = [ctypes.c_int] + sloargs.Slo_GetAnnotationKeyById.restype = ctypes.c_char_p + sloargs.Slo_GetAnnotationByKey.argtypes = [ctypes.c_char_p] + sloargs.Slo_GetAnnotationByKey.restype = ctypes.c_char_p + + +class _SloArgs_Aqsis(_SloArgs): + def __init__(self): + _SloArgs.__init__(self, libName="slxargs", VISSYMDEF=_VISSYMDEF_aqsis) + + + +# Pixie + +class _UDefaultVal(ctypes.Union): + pass +_UDefaultVal._fields_ = [("matrix", ctypes.POINTER(16*ctypes.c_float)), + ("vector", ctypes.POINTER(3*ctypes.c_float)), + ("scalar", ctypes.c_float), + ("string", ctypes.c_char_p), + ("array", ctypes.POINTER(_UDefaultVal)) + ] + +class _TSdrParameter(ctypes.Structure): + pass +_TSdrParameter._fields_ = [("name", ctypes.c_char_p), + ("type", ctypes.c_int), + ("container", ctypes.c_int), + ("writable", ctypes.c_int), + ("numItems", ctypes.c_int), + ("space", ctypes.c_char_p), + ("defaultValue", _UDefaultVal), + ("next", ctypes.POINTER(_TSdrParameter)) + ] + +class _TSdrShader(ctypes.Structure): + _fields_ = [("name", ctypes.c_char_p), + ("type", ctypes.c_int), + ("parameters", ctypes.POINTER(_TSdrParameter)) + ] + +class _SloArgs_Pixie(_SloArgs): + """Pixie version. + + This class does derive from _SloArgs but doesn't use the sloargs interface + as Pixie has its own interface. + """ + + def __init__(self): + _SloArgs.__init__(self, libName="sdr", VISSYMDEF=None) + self._shaderType = {0:"surface", 1:"displacement", 2:"volume", 3:"light", 4:"imager"} + self._container = {0:"constant", 1:"uniform", 2:"varying", 3:"vertex"} + self._paramType = {0:"float", 1:"vector", 2:"normal", 3:"point", 4:"color", 5:"matrix", 6:"string"} + + def _declareFunctions(self, sloargs): + # We don't call the inherited method as Pixie's library does not + # provide the same interface than PRMan + + sloargs.sdrGet.argtypes = [ctypes.c_char_p] + sloargs.sdrGet.restype = ctypes.POINTER(_TSdrShader) + sloargs.sdrDelete.argtypes = [ctypes.POINTER(_TSdrShader)] + + def getShaderInfo(self, shader): + sdr = self._sloargs + shaderInfo = sdr.sdrGet(shader) + if not shaderInfo: + raise IOError('Failed to open shader "%s"'%shader) + + try: + shaderName = shaderInfo.contents.name + shaderType = self._shaderType.get(shaderInfo.contents.type, "?") + params = [] + param = shaderInfo.contents.parameters + while param: + param = param.contents + + if param.writable: + output = "output" + else: + output = "" + + storage = self._container.get(param.container, "?") + paramType = self._paramType.get(param.type, "?") + arrLen = param.numItems + if arrLen==1: + arrLen = None + name = param.name + space = param.space + if arrLen==None: + defaultVal = self._getDefaultVal(paramType, param.defaultValue) + else: + defaultVal = [] + for i in range(arrLen): + defaultVal.append(self._getDefaultVal(paramType, param.defaultValue.array[i])) + + params.append((output,storage,paramType,arrLen,name,space,defaultVal)) + param = param.next + finally: + sdr.sdrDelete(shaderInfo) + + # Pixie reports the parameters in reverse order, so reverse the list + params.reverse() + return [(shaderType, shaderName, params)] + + def _getDefaultVal(self, paramType, defval): + if paramType=="float": + return defval.scalar + elif paramType in ["vector", "normal", "point", "color"]: + return tuple(defval.vector.contents) + elif paramType=="string": + defaultVal = defval.string + elif paramType=="matrix": + return tuple(defval.matrix.contents) + return None + +############################################################################ + +# The registered SloArgs classes. +# Key: Compiled Shader Suffix - Value: SloArgs class +_sloArgsClasses = {} + +# The already instantiated SloArgs objects (so that they can be reused). +# Key: Compiled Shader Suffix - Value: SloArgs instance +_sloArgsInstances = {} + +def registerSloArgs(sloSuffix, sloArgsCls): + """Register a SloArgs class for a particular renderer. + + sloSuffix is the suffix (without dot) that is used for the compiled + shaders of this renderer. sloArgsCls is the SloArgs class that + can read compiled shaders for this renderer. + """ + global _sloArgsClasses + + sloSuffix = sloSuffix.lower() + _sloArgsClasses[sloSuffix] = sloArgsCls + +def slparams(shader): + """Read shader parameters. + """ + global _sloArgsClasses, _sloArgsInstances + + if not os.path.exists(shader): + raise IOError('Shader file "%s" does not exist'%shader) + + # Get the SloArgs class for the renderer that produced the compiled + # shader (based on the shader suffix). + shaderBase,ext = os.path.splitext(shader) + ext = ext.lower()[1:] + if ext in _sloArgsInstances: + sloArgs = _sloArgsInstances[ext] + elif ext in _sloArgsClasses: + SloArgs = _sloArgsClasses[ext] + sloArgs = SloArgs() + _sloArgsInstances[ext] = sloArgs + else: + raise ValueError, "Unknown compiled shader extension: %s"%shader + + return sloArgs.getShaderInfo(shader) + + +# Register the built-in SloArgs classes +registerSloArgs("slo", _SloArgs_PRMan) +registerSloArgs("sdl", _SloArgs_3Delight) +registerSloArgs("slx", _SloArgs_Aqsis) +registerSloArgs("sdr", _SloArgs_Pixie) + +### TEST ### +if __name__=="__main__": + import cgkit.slparams + + if 0: + info = cgkit.slparams.slparams("foo.sl") + if len(info)>0: + type,name,params = info[0] + print "Shader:",name + print "Type :",type + for param in params: + print param, cgkit.slparams.convertdefault(param) + + info = slparams("foo.sdr") + if len(info)>0: + type,name,params = info[0] + print "Shader:",name + print "Type :",type + for param in params: + print param Modified: cgkit/trunk/cgkit/slparams.py =================================================================== --- cgkit/trunk/cgkit/slparams.py 2008-09-14 09:07:04 UTC (rev 253) +++ cgkit/trunk/cgkit/slparams.py 2008-09-14 09:24:20 UTC (rev 254) @@ -52,6 +52,7 @@ import os, os.path, sys, string, StringIO, sltokenize, types import cgtypes, math, sl, simplecpp import _slparser +import sloargs class SLParamsError(Exception): pass @@ -219,12 +220,12 @@ # slparams def slparams(slfile=None, cpp=None, cpperrstream=sys.stderr, slname=None, includedirs=None, defines=None): - """Extracts the shader parameters from a Shading Language source file. + """Extracts the shader parameters from a RenderMan Shader file. - The argument slfile is either the name of the shader source file - (*.sl) or it is a file-like object that provides the shader - sources. cpp determines how the shader source is preprocessed. It - can either be a string containing the name of an external + The argument slfile is either the name of a compiled shader, the name of + the shader source file (*.sl) or a file-like object that provides the + shader sources. cpp determines how the shader source is preprocessed. + It can either be a string containing the name of an external preprocessor tool (such as 'cpp') that must take the file name as parameter and dump the preprocessed output to stdout or it can be a callable that takes slfile and cpperrstream as input and returns @@ -259,10 +260,16 @@ - The default value (always given as a string) """ - if slname!=None: + if slname is not None: slfile = slname - if slfile==None: + if slfile is None: return [] + + # Check if the input file is a string referring to a compiled shader + # (suffix != .sl). If so, use the sloargs module to get the shader information + if isinstance(slfile, basestring): + if os.path.splitext(slfile)[1].lower()!=".sl": + return sloargs.slparams(slfile) # Run the preprocessor on the input file... Modified: cgkit/trunk/changelog.txt =================================================================== --- cgkit/trunk/changelog.txt 2008-09-14 09:07:04 UTC (rev 253) +++ cgkit/trunk/changelog.txt 2008-09-14 09:24:20 UTC (rev 254) @@ -3,8 +3,14 @@ Changes from cgkit-2.0.0alpha8 to cgkit-2.0.0alpha9: +New features: + +- New module sloargs to parse compiled RenderMan shaders. + Bug fixes/enhancements: +- slparams: The module can now also retrieve information from compiled shaders + (from various renderers). - ObjMaterial: Texture maps were rendered with a filter width of 0.01 which doesn't make much sense. For now, I set it to 1.0, but this should probably be customizable. This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. 
 [cgkit-commits] SF.net SVN: cgkit:[256] cgkit/trunk From: - 2008-09-16 05:17:42 Revision: 256 http://cgkit.svn.sourceforge.net/cgkit/?rev=256&view=rev Author: mbaas Date: 2008-09-16 12:17:39 +0000 (Tue, 16 Sep 2008) Log Message: ----------- Fixed compiler warnings (due to a missing virtual destructor (but all destructors have been empty anyway)). Modified Paths: -------------- cgkit/trunk/supportlib/include/massproperties.h cgkit/trunk/wrappers/rply/py_rply_write.cpp Modified: cgkit/trunk/supportlib/include/massproperties.h =================================================================== --- cgkit/trunk/supportlib/include/massproperties.h 2008-09-16 12:16:17 UTC (rev 255) +++ cgkit/trunk/supportlib/include/massproperties.h 2008-09-16 12:17:39 UTC (rev 256) @@ -14,7 +14,7 @@ /* TODO: - Normal calculation for polygons with more than 3 vertices - Error checking for degenerate faces - - Error checking for zero volume (DivisionByZero) + - Error checking for zero volume (DivisionByZero) */ #ifndef VOLINT_H @@ -69,6 +69,7 @@ { public: FaceEnum() {} + virtual ~FaceEnum() {} virtual bool next(FACE& face) = 0; }; Modified: cgkit/trunk/wrappers/rply/py_rply_write.cpp =================================================================== --- cgkit/trunk/wrappers/rply/py_rply_write.cpp 2008-09-16 12:16:17 UTC (rev 255) +++ cgkit/trunk/wrappers/rply/py_rply_write.cpp 2008-09-16 12:17:39 UTC (rev 256) @@ -58,6 +58,8 @@ class Writer { public: + virtual ~Writer() {} + /** Write the data item at position \a idx into the file. */ This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. 
 [cgkit-commits] SF.net SVN: cgkit:[266] cgkit/trunk From: - 2009-01-07 21:58:42 Revision: 266 http://cgkit.svn.sourceforge.net/cgkit/?rev=266&view=rev Author: mbaas Date: 2009-01-07 21:32:31 +0000 (Wed, 07 Jan 2009) Log Message: ----------- The frame block now uses the frame number from the timer object (instead of 0). Modified Paths: -------------- cgkit/trunk/cgkit/ribexport.py cgkit/trunk/changelog.txt Modified: cgkit/trunk/cgkit/ribexport.py =================================================================== --- cgkit/trunk/cgkit/ribexport.py 2009-01-07 21:31:12 UTC (rev 265) +++ cgkit/trunk/cgkit/ribexport.py 2009-01-07 21:32:31 UTC (rev 266) @@ -109,6 +109,7 @@ scene = getScene() self.timestep = scene.timer().timestep + frameNr = int(round(scene.timer().frame)) self.bake = bake # A list with all light sources in the scene @@ -266,7 +267,7 @@ if isinstance(rpass, TexPass): # nr+=1 # print "\015Pass %d..."%nr, - rpass.doPass(0) + rpass.doPass(frameNr) rpass._done = True # Other passes... @@ -275,7 +276,7 @@ continue # nr+=1 # print "\015Pass %d..."%nr, - rpass.doPass(0) + rpass.doPass(frameNr) rpass._done = True RiEnd() @@ -1531,7 +1532,7 @@ } } - """ +""" def shaderParams(self, passes): """Return a dictionary with shader parameters and their values.""" @@ -1589,7 +1590,7 @@ Cl = att*intensity*color "rgb" (1,1,1); } } - """ +""" def shaderParams(self, passes): """Return a dictionary with shader parameters and their values.""" @@ -1664,7 +1665,7 @@ } } - """ +""" def shaderParams(self, passes): """Return a dictionary with shader parameters and their values.""" @@ -1716,7 +1717,7 @@ Cl = intensity * color "rgb" (1,1,1); } } - """ +""" def shaderParams(self, passes): """Return a dictionary with shader parameters and their values.""" @@ -1877,7 +1878,7 @@ } BAKE_END } - """ +""" def surfaceShaderParams(self, passes): """Return a dictionary with shader parameters and their values.""" Modified: cgkit/trunk/changelog.txt =================================================================== --- cgkit/trunk/changelog.txt 2009-01-07 21:31:12 UTC (rev 265) +++ cgkit/trunk/changelog.txt 2009-01-07 21:32:31 UTC (rev 266) @@ -24,6 +24,8 @@ the module. An error is only triggered when one of the functions gets called. - rply: The sources can now also be compiled using a C++ compiler. Thanks to Chris Foster for the patch! +- ribexport: The Frame block now uses the frame number from the timer object + (instead of always using 0). - ribexport: Added RenderMan support for the torus geometry. - ri/cri: Passing unicode strings in a parameter list has caused an exception. This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. 
 [cgkit-commits] SF.net SVN: cgkit:[267] cgkit/trunk From: - 2009-01-19 22:39:26 Revision: 267 http://cgkit.svn.sourceforge.net/cgkit/?rev=267&view=rev Author: mbaas Date: 2009-01-19 22:14:41 +0000 (Mon, 19 Jan 2009) Log Message: ----------- Improved library search functionality. Modified Paths: -------------- cgkit/trunk/cgkit/_cri.py cgkit/trunk/cgkit/cri.py cgkit/trunk/changelog.txt Added Paths: ----------- cgkit/trunk/cgkit/rmanlibutil.py Modified: cgkit/trunk/cgkit/_cri.py =================================================================== --- cgkit/trunk/cgkit/_cri.py 2009-01-07 21:32:31 UTC (rev 266) +++ cgkit/trunk/cgkit/_cri.py 2009-01-19 22:14:41 UTC (rev 267) @@ -43,7 +43,7 @@ import os.path from ctypes import * -import ctypes.util +import rmanlibutil def loadRI(libName): """Load a RenderMan library and return a module-like handle to it. @@ -58,10 +58,7 @@ """ # Try to figure out the location of the lib if the name is not an absolute path... - if not os.path.isabs(libName): - p = ctypes.util.find_library(libName) - if p is not None: - libName = p + libName = rmanlibutil.resolveRManLib(libName) # Load the library... ri = cdll.LoadLibrary(libName) Modified: cgkit/trunk/cgkit/cri.py =================================================================== --- cgkit/trunk/cgkit/cri.py 2009-01-07 21:32:31 UTC (rev 266) +++ cgkit/trunk/cgkit/cri.py 2009-01-19 22:14:41 UTC (rev 267) @@ -1408,7 +1408,7 @@ # v=scalar? if vtype in ScalarTypes: res.append(v) - # no stadard scalar or string. Then it might be a sequence.. + # no standard scalar or string. Then it might be a sequence.. else: # Check if it is really a sequence... try: Added: cgkit/trunk/cgkit/rmanlibutil.py =================================================================== --- cgkit/trunk/cgkit/rmanlibutil.py (rev 0) +++ cgkit/trunk/cgkit/rmanlibutil.py 2009-01-19 22:14:41 UTC (rev 267) @@ -0,0 +1,148 @@ +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is the Python Computer Graphics Kit. +# +# The Initial Developer of the Original Code is Matthias Baas. +# Portions created by the Initial Developer are Copyright (C) 2009 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** +# $Id: riutil.py,v 1.1.1.1 2004/12/12 14:31:21 mbaas Exp$ + +import sys, os, os.path +import ctypes.util + +def resolveRManLib(libName, renderer=None): + """Resolve the given library name. + + If the name is an absolute file name, it is just returned unmodified. + Otherwise the method tries to resolve the name and return an absolute + path to the library. If no library file could be found, the name + is returned unmodified. + + renderer may be the name of a renderer package (aqsis, pixie, 3delight, + prman) which serves as a hint to which package the library belongs to. + If not given, the renderer is determined from the library name. + """ + if os.path.isabs(libName): + return libName + + # Try to figure out the location of the lib + lib = ctypes.util.find_library(libName) + if lib is not None: + return lib + + # A list of library search paths... + searchPaths = [] + + # Is there a renderer-specific search path? + if renderer is None: + renderer = rendererFromLib(libName) + libDir = rendererLibDir(renderer) + if libDir is not None: + searchPaths.append(libDir) + + # Also examine LD_LIBRARY_PATH if we are on Linux + if sys.platform.startswith("linux"): + libPaths = os.getenv("LD_LIBRARY_PATH") + if libPaths is not None: + searchPaths.extend(libPaths.split(":")) + + # Check the search paths... + libFileName = libraryFileName(libName) + for path in searchPaths: + lib = os.path.join(path, libFileName) + if os.path.exists(lib): + return lib + + # Nothing found, then just return the original name + return libName + +def rendererFromLib(libName): + """Return the renderer package name given a library name. + + libName is the name of a RenderMan library. The function tries to + determine from which renderer it is and returns the name of the + render package. None is returned if the library name is unknown. + """ + name = os.path.basename(libName) + name = os.path.splitext(name)[0] + if name.startswith("lib"): + name = name[3:] + + if name in ["aqsislib", "ri2rib", "slxargs"]: + return "aqsis" + elif libName in ["sdr", "ri"]: + return "pixie" + elif libName in ["3delight"]: + return "3delight" + elif libName in ["prman"]: + return "prman" + + return None + +def rendererLibDir(renderer): + """Return a renderer-specific library path. + + The return path is based on a renderer-specific environment variable. + None is returned when no path could be determined. + renderer may be None, "aqsis", "pixie", "3delight" or "prman" (case-insensitive). + """ + if renderer is None: + return None + + envVarDict = {"aqsis":"AQSISHOME", + "pixie":"PIXIEHOME", + "3delight":"DELIGHT", + "prman":"RMANTREE"} + + envVar = envVarDict.get(renderer.lower()) + if envVar is not None: + base = os.getenv(envVar) + if base is not None: + return os.path.join(base, "lib") + return None + +def libraryFileName(libName): + """Extend a base library name to a file name. + + Example: "foo" -> "libfoo.so" (Linux) + -> "foo.dll" (Windows) + -> "libfoo.dylib" (OSX) + """ + if sys.platform.startswith("linux"): + return "lib%s.so"%libName + elif sys.platform=="darwin": + return "lib%s.dylib"%libName + elif sys.platform.startswith("win"): + return "%s.dll"%libName + return libName + +##################################################################### +if __name__=='__main__': + pass Modified: cgkit/trunk/changelog.txt =================================================================== --- cgkit/trunk/changelog.txt 2009-01-07 21:32:31 UTC (rev 266) +++ cgkit/trunk/changelog.txt 2009-01-19 22:14:41 UTC (rev 267) @@ -27,6 +27,8 @@ - ribexport: The Frame block now uses the frame number from the timer object (instead of always using 0). - ribexport: Added RenderMan support for the torus geometry. +- cri: Improved library detection (uses renderer-specific environment variables + to locate the shared RenderMan library) - ri/cri: Passing unicode strings in a parameter list has caused an exception. ---------------------------------------------------------------------- This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. 
 [cgkit-commits] SF.net SVN: cgkit:[268] cgkit/trunk From: - 2009-01-25 18:46:05 Revision: 268 http://cgkit.svn.sourceforge.net/cgkit/?rev=268&view=rev Author: mbaas Date: 2009-01-25 18:46:00 +0000 (Sun, 25 Jan 2009) Log Message: ----------- The return values of slparams/sloargs are now special objects instead of tuples. The shader meta data can now also be returned. Modified Paths: -------------- cgkit/trunk/cgkit/sloargs.py cgkit/trunk/cgkit/slparams.py cgkit/trunk/changelog.txt cgkit/trunk/unittests/data/testshader.sl cgkit/trunk/unittests/test_slparams.py Added Paths: ----------- cgkit/trunk/cgkit/_slreturntypes.py Added: cgkit/trunk/cgkit/_slreturntypes.py =================================================================== --- cgkit/trunk/cgkit/_slreturntypes.py (rev 0) +++ cgkit/trunk/cgkit/_slreturntypes.py 2009-01-25 18:46:00 UTC (rev 268) @@ -0,0 +1,119 @@ +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is the Python Computer Graphics Kit. +# +# The Initial Developer of the Original Code is Matthias Baas. +# Portions created by the Initial Developer are Copyright (C) 2008 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +"""This module contains the return types used by the slparams and sloargs module. +""" + +class _ShaderInfo: + """Holds information about a shader. + """ + + def __init__(self, type=None, name=None, params=None, meta=None): + """Constructor. + """ + if params is None: + params = [] + if meta is None: + meta = {} + + # The shader type (surface, displacement, ...) + self.type = type + # The shader name + self.name = name + # The shader parameters (a list of _ShaderParam objects) + self.params = params + # The meta data attached to the shader (a dictionary) + self.meta = meta + + def __str__(self): + return "(%r, %r, %s)"%(self.type, self.name, self.params) + + __repr__ = __str__ + + def __len__(self): + return 3 + + def __iter__(self): + yield self.type + yield self.name + yield self.params + + def __getitem__(self, idx): + return (self.type, self.name, self.params)[idx] + + +class _ShaderParam: + """Holds information about a shader parameter. + """ + + def __init__(self, outputSpec=None, storage=None, type=None, size=None, name=None, space=None, default=None): + """Constructor. + """ + + # The output specifier ("output" or empty string) + self.outputSpec = outputSpec + # The storage class ("uniform", "varying") + self.storage = storage + # The parameter type + self.type = type + # The array length or None if the param is not an array + self.size = size + # The parameter name + self.name = name + # The space (or list of spaces in case of an array) or None + self.space = space + # The default value + self.default = default + + def __str__(self): + return "(%r, %r, %r, %r, %r, %r, %r)"%(self.outputSpec, self.storage, self.type, self.size, self.name, self.space, self.default) + + __repr__ = __str__ + + def __len__(self): + return 7 + + def __iter__(self): + yield self.outputSpec + yield self.storage + yield self.type + yield self.size + yield self.name + yield self.space + yield self.default + + def __getitem__(self, idx): + return (self.outputSpec, self.storage, self.type, self.size, self.name, self.space, self.default)[idx] + Modified: cgkit/trunk/cgkit/sloargs.py =================================================================== --- cgkit/trunk/cgkit/sloargs.py 2009-01-19 22:14:41 UTC (rev 267) +++ cgkit/trunk/cgkit/sloargs.py 2009-01-25 18:46:00 UTC (rev 268) @@ -36,7 +36,11 @@ import sys, os, os.path import ctypes import ctypes.util +from _slreturntypes import _ShaderInfo, _ShaderParam +import rmanlibutil +#################### ctypes type declarations ####################### + class _POINT(ctypes.Structure): _fields_ = [("xval", ctypes.c_float), ("yval", ctypes.c_float), @@ -84,8 +88,9 @@ ("arraylen", ctypes.c_int), ("default", _DEFAULTVAL), ] - +######################## Base class ############################## + class _SloArgs: """Provides functionality to read shader parameters using the C sloargs interface. @@ -107,7 +112,7 @@ self._VISSYMDEF = VISSYMDEF - libName = self._resolveLibraryName(libName) + libName = rmanlibutil.resolveRManLib(libName) self.libName = libName # Load the library... @@ -124,68 +129,6 @@ def defaultLibName(): raise NotImplementedError("defaultLibName() must be implemented by derived classes") - def _rendererLibDir(self): - """Return a renderer-specific library path where the sloargs lib is searched for. - - Derived classes should implement this and examine renderer-specific environment - variables. - """ - pass - - def _resolveLibraryName(self, libName): - """Resolve the given library name. - - If the name is an absolute file name, it is just returned unmodified. - Otherwise the method tries to resolve the name and return an absolute - path to the library. If no library file could be found, the name - is returned unmodified. - """ - if os.path.isabs(libName): - return libName - - # Try to figure out the location of the lib - lib = ctypes.util.find_library(libName) - if lib is not None: - return lib - - # A list of library search paths... - searchPaths = [] - - # Is there a renderer-specific search path? - libDir = self._rendererLibDir() - if libDir is not None: - searchPaths.append(libDir) - - # Also examine LD_LIBRARY_PATH if we are on Linux - if sys.platform.startswith("linux"): - libPaths = os.getenv("LD_LIBRARY_PATH") - if libPaths is not None: - searchPaths.extend(libPaths.split(":")) - - # Check the search paths... - for path in searchPaths: - lib = os.path.join(path, self._libFileName(libName)) - if os.path.exists(lib): - return lib - - # Nothing found, then just return the original name - return libName - - def _libFileName(self, libName): - """Extend a base library name to a file name. - - Example: "foo" -> "libfoo.so" (Linux) - -> "foo.dll" (Windows) - -> "libfoo.dylib" (OSX) - """ - if sys.platform.startswith("linux"): - return "lib%s.so"%libName - elif sys.platform=="darwin": - return "lib%s.dylib"%libName - elif sys.platform.startswith("win"): - return "%s.dll"%libName - return libName - def _loadLibrary(self, libName): """Load the library providing the sloargs interface. @@ -228,6 +171,15 @@ sloargs.Slo_StortoStr.restype = ctypes.c_char_p sloargs.Slo_DetailtoStr.argtypes = [ctypes.c_int] sloargs.Slo_DetailtoStr.restype = ctypes.c_char_p + + def _getMetaData(self): + """Return the meta data for the current shader. + + The function is called after Slo_SetShader() but before + Slo_EndShader() has been called. The return value should + be the meta data for the shader (preferably as a dict). + """ + return None def getShaderInfo(self, shader): """Read the shader parameters from a given shader. @@ -247,6 +199,7 @@ shaderName = sloargs.Slo_GetName() shaderType = sloargs.Slo_TypetoStr(sloargs.Slo_GetType()) + metaData = self._getMetaData() params = [] numParams = sloargs.Slo_GetNArgs() for i in range(1, numParams+1): @@ -267,10 +220,10 @@ if space=="": space = None defaultVal = self._getDefaultVal(symdef) - params.append((output,storage,paramType,arrLen,name,space,defaultVal)) + params.append(_ShaderParam(output,storage,paramType,arrLen,name,space,defaultVal)) sloargs.Slo_EndShader() - return [(shaderType, shaderName, params)] + return [_ShaderInfo(type=shaderType, name=shaderName, params=params, meta=metaData)] def _getSpace(self, symdef): if symdef.arraylen>0: @@ -324,6 +277,8 @@ return None +############################# PRMan ################################## + class _SloArgs_PRMan(_SloArgs): def __init__(self, libName): _SloArgs.__init__(self, libName=libName, VISSYMDEF=_VISSYMDEF_prman) @@ -332,14 +287,8 @@ def defaultLibName(): return "prman" - def _rendererLibDir(self): - base = os.getenv("RMANTREE") - if base is not None: - return os.path.join(base, "lib") - else: - return None +############################# 3Delight ################################## - class _SloArgs_3Delight(_SloArgs): def __init__(self, libName): _SloArgs.__init__(self, libName=libName, VISSYMDEF=_VISSYMDEF_3delight) @@ -348,13 +297,6 @@ def defaultLibName(): return "3delight" - def _rendererLibDir(self): - base = os.getenv("DELIGHT") - if base is not None: - return os.path.join(base, "lib") - else: - return None - def _declareFunctions(self, sloargs): _SloArgs._declareFunctions(self, sloargs) # 3Delight-specific functions @@ -365,7 +307,21 @@ sloargs.Slo_GetAnnotationByKey.argtypes = [ctypes.c_char_p] sloargs.Slo_GetAnnotationByKey.restype = ctypes.c_char_p + def _getMetaData(self): + res = {} + n = self._sloargs.Slo_GetNAnnotations() + for i in range(1, n+1): + key = self._sloargs.Slo_GetAnnotationKeyById(i) + if key is None: + continue + ann = self._sloargs.Slo_GetAnnotationByKey(key) + if ann is None: + continue + res[key] = ann + return res +############################# Aqsis ################################## + class _SloArgs_Aqsis(_SloArgs): def __init__(self, libName): _SloArgs.__init__(self, libName=libName, VISSYMDEF=_VISSYMDEF_aqsis) @@ -375,9 +331,8 @@ return "slxargs" +############################# Pixie ################################## -# Pixie - class _UDefaultVal(ctypes.Union): pass _UDefaultVal._fields_ = [("matrix", ctypes.POINTER(16*ctypes.c_float)), @@ -422,13 +377,6 @@ def defaultLibName(): return "sdr" - def _rendererLibDir(self): - base = os.getenv("PIXIEHOME") - if base is not None: - return os.path.join(base, "lib") - else: - return None - def _declareFunctions(self, sloargs): # We don't call the inherited method as Pixie's library does not # provide the same interface than PRMan @@ -551,6 +499,8 @@ def slparams(shader): """Read shader parameters. + + See slparams.slparams() for more details. """ global _sloArgsClasses, _sloArgsInstances, _sloLibNames Modified: cgkit/trunk/cgkit/slparams.py =================================================================== --- cgkit/trunk/cgkit/slparams.py 2009-01-19 22:14:41 UTC (rev 267) +++ cgkit/trunk/cgkit/slparams.py 2009-01-25 18:46:00 UTC (rev 268) @@ -52,6 +52,7 @@ import os, os.path, sys, string, StringIO, sltokenize, types import cgtypes, math, sl, simplecpp import _slparser +from _slreturntypes import _ShaderInfo, _ShaderParam import sloargs class SLParamsError(Exception): @@ -144,13 +145,13 @@ if self.arraylen==None: if self.space==None: self.space = self.defaultSpace(self.type) - self.params.append((self.output, self.detail, self.type, None, + self.params.append(_ShaderParam(self.output, self.detail, self.type, None, self.name, self.space, self.default)) else: spaces = self.spaces if self.defaultSpace(self.type)==None: spaces = None - self.params.append((self.output, self.detail, self.type, + self.params.append(_ShaderParam(self.output, self.detail, self.type, self.arraylen, self.name, spaces, self.default)) self.arraylen = None self.name = "" @@ -245,19 +246,30 @@ look for include files. defines is a list of tuples (name, value) that specify the predefined symbols to use. - The function returns a list of 3-tuples, one for each shader found - in the file. The tuple contains the type, the name and the - parameters of the shader. The parameters are given as a list of - 7-tuples describing each parameter. The tuple contains the - following information (in the given order): + The function returns a list of shader info objects. These objects have + 4 attributes: + + - type: The type of the shader (surface, displacement, etc.) + - name: The name of the shader + - params: The shader parameters (see below) + - meta: The shader meta data + + The parameters are given as a list of shader parameter objects + describing each parameter. A shader parameter object has the + following attributes: - - The output specifier (either "output" or an empty string) - - The storage class ("uniform" or "varying") - - The parameter type - - The array length or None if the parameter is not an array - - The name of the parameter - - The space in which a point-like type was defined - - The default value (always given as a string) + - outputSpec: The output specifier (either "output" or an empty string) + - storage: The storage class ("uniform" or "varying") + - type: The parameter type + - size: The array length or None if the parameter is not an array + - name: The name of the parameter + - spacE: The space in which a point-like type was defined + - default: The default value (always given as a string) + + For backwards compatibility, the shader info object behaves like a + 3-tuple (type, name, params). The meta data can only be accessed via name + though. The shader parameter objects can also be used like 7-tuples + containing the above data (in the order given above). """ if slname is not None: @@ -291,7 +303,9 @@ # return wrap_error_reporter(parser, "definitions") try: - return getattr(parser, "definitions")() + lst = getattr(parser, "definitions")() + # Turn the 3-tuples into _ShaderInfo objects + return map(lambda tup: _ShaderInfo(*tup), lst) except _slparser.NoMoreTokens, err: raise NoMoreTokens, "No more tokens" except _slparser.SyntaxError, err: Modified: cgkit/trunk/changelog.txt =================================================================== --- cgkit/trunk/changelog.txt 2009-01-19 22:14:41 UTC (rev 267) +++ cgkit/trunk/changelog.txt 2009-01-25 18:46:00 UTC (rev 268) @@ -13,6 +13,9 @@ channelBox option (+ some other bugfixes). - slparams: The module can now also retrieve information from compiled shaders (from various renderers). +- slparams: The returned shader info/shader parameters are no tuples anymore + but special objects whose attributes hold the return values. For backwards, + compatibility, those objects can still be used like tuples. - ObjMaterial: Texture maps were rendered with a filter width of 0.01 which doesn't make much sense. For now, I set it to 1.0, but this should probably be customizable. Modified: cgkit/trunk/unittests/data/testshader.sl =================================================================== --- cgkit/trunk/unittests/data/testshader.sl 2009-01-19 22:14:41 UTC (rev 267) +++ cgkit/trunk/unittests/data/testshader.sl 2009-01-25 18:46:00 UTC (rev 268) @@ -4,6 +4,8 @@ #define arg float Kd = 0.5 +#pragma annotation "MyData" "MyValue" + surface testshader(float Ka = 1; varying vector norm = 0; float uv[2] = {1,2}; /* comment */ Modified: cgkit/trunk/unittests/test_slparams.py =================================================================== --- cgkit/trunk/unittests/test_slparams.py 2009-01-19 22:14:41 UTC (rev 267) +++ cgkit/trunk/unittests/test_slparams.py 2009-01-25 18:46:00 UTC (rev 268) @@ -16,10 +16,11 @@ def testCompiledShader(self): """Test reading params from compiled shaders. """ - renderers = [#("3Delight", "shaderdl", "sdl"), + renderers = [("3Delight", "shaderdl", "sdl"), #("Aqsis", "aqsl", "slx"), #("Pixie", "sdrc", "sdr"), - ("PRMan", "shader", "slo")] + #("PRMan", "shader", "slo") + ] for data in renderers: renderer,shaderCompiler,shaderExt = data slcName = "tmp/testshader.%s"%shaderExt @@ -34,6 +35,11 @@ continue res = slparams.slparams(slcName) self.assertTestShaderParams(res) + + # Check the meta data (renderer-specific) + if renderer=="3Delight": + meta = res[0].meta + self.assertEqual(meta, {"MyData": "MyValue"}) def assertTestShaderParams(self, shaderInfo): """Check the result from slparams(). This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. 
 [cgkit-commits] SF.net SVN: cgkit:[281] cgkit/trunk From: - 2009-02-08 14:36:52 Revision: 281 http://cgkit.svn.sourceforge.net/cgkit/?rev=281&view=rev Author: mbaas Date: 2009-02-08 14:36:48 +0000 (Sun, 08 Feb 2009) Log Message: ----------- Added support for reading/writing multiple points at once. Modified Paths: -------------- cgkit/trunk/cgkit/pointcloud.py cgkit/trunk/setup.py Added Paths: ----------- cgkit/trunk/pyrex/ cgkit/trunk/pyrex/c/ cgkit/trunk/pyrex/c/_pointcloud.c cgkit/trunk/pyrex/pyx/ cgkit/trunk/pyrex/pyx/_pointcloud.pyx cgkit/trunk/unittests/test_pointcloud.py Modified: cgkit/trunk/cgkit/pointcloud.py =================================================================== --- cgkit/trunk/cgkit/pointcloud.py 2009-02-06 17:14:25 UTC (rev 280) +++ cgkit/trunk/cgkit/pointcloud.py 2009-02-08 14:36:48 UTC (rev 281) @@ -34,13 +34,62 @@ # ***** END LICENSE BLOCK ***** # $Id: riutil.py,v 1.1.1.1 2004/12/12 14:31:21 mbaas Exp$ -# Point cloud test - import os.path import ctypes import rmanlibutil +import _pointcloud +try: + import numpy + _numpy_available = True +except ImportError: + _numpy_available = False +def _arrayPointer(a, n): + """Check an array and return a pointer to its start. + + a is an array object and n the minimum number of values the array + must contain. An exception is thrown when the array is too small + or its type is not float (4 bytes) or when a is not a recognized + array at all. + """ + # Is the array a ctypes array? + if isinstance(a, ctypes.Array): + if a._type_!=ctypes.c_float: + raise TypeError("Float array expected") + if len(a)0: + n = min(batchSize, num) + if bufLen!=n: + if combinedBuffer: + if numpyArray: + buffer = numpy.zeros(shape=(n,7+self.datasize), dtype=numpy.float32) + else: + buffer = (((7+self.datasize)*n)*ctypes.c_float)() + else: + if numpyArray: + ps = numpy.zeros(shape=(n,3), dtype=numpy.float32) + ns = numpy.zeros(shape=(n,3), dtype=numpy.float32) + rs = numpy.zeros(shape=(n,), dtype=numpy.float32) + ds = numpy.zeros(shape=(n,self.datasize), dtype=numpy.float32) + else: + ps = (3*n*ctypes.c_float)() + ns = (3*n*ctypes.c_float)() + rs = (n*ctypes.c_float)() + ds = (self.datasize*n*ctypes.c_float)() + buffer = (ps,ns,rs,ds) + bufLen = n + self.readDataPoints(n, buffer) + yield buffer + num -= n + def _loadPtcLib(self, libName): """Load a RenderMan library providing the point cloud API. """ @@ -190,11 +355,17 @@ class PtcWriter: - def __init__(self, fileName, vars, world2eye, world2ndc, format, libName): + """Point cloud writer class. + """ + + def __init__(self, fileName, libName, vars, world2eye, world2ndc, format): """Constructor. fileName is the name of the point cloud file and libName the name of the shared library that implements the point cloud API. + vars is a list of tuples (type, name) that specifies what additional + variables to write into the file. world2eye and world2ndc are 4x4 + matrices and format a tuple """ self._handle = None @@ -207,11 +378,8 @@ xres,yres,aspect = format - m = (16*ctypes.c_float)() - m[0] = 1.0 - m[5] = 1.0 - m[10] = 1.0 - m[15] = 1.0 + w2e = self._matrixToCTypes(world2eye) + w2n = self._matrixToCTypes(world2ndc) n = len(vars) cvartypes = (n*ctypes.c_char_p)() @@ -234,23 +402,41 @@ else: raise RuntimeError("Unknown point cloud variable type: %s"%type) - self._dataSize = idx + self.datasize = idx self._dataInitCode = code cformat = (3*ctypes.c_float)(float(xres), float(yres), float(aspect)) - handle = ptclib.PtcCreatePointCloudFile(fileName, n, cvartypes, cvarnames, m, m, cformat) + handle = ptclib.PtcCreatePointCloudFile(fileName, n, cvartypes, cvarnames, w2e, w2n, cformat) if handle is None: raise IOError("Cannot open point cloud file %s for writing"%fileName) self._handle = handle + def _matrixToCTypes(self, m): + """Convert a matrix into a ctypes array. + + m can be any object that contains 16 floats (the values may be nested). + """ + values = [] + for v in m: + try: + values.extend(list(v)) + except: + values.append(v) + if len(values)!=16: + raise ValueError("Matrix must be composed of 16 values, got %s instead."%len(values)) + return (16*ctypes.c_float)(*values) + def __del__(self): self.close() def close(self): + """Close the point cloud file. + + This method is also called from the destructor. + """ if self._handle is not None: - print "CLOSE ptc file (write)" self._PtcFinishPointCloudFile(self._handle) self._handle = None @@ -267,12 +453,51 @@ p = (3*ctypes.c_float)(*tuple(point)) n = (3*ctypes.c_float)(*tuple(normal)) - cdata = (self._dataSize*ctypes.c_float)() + cdata = (self.datasize*ctypes.c_float)() exec self._dataInitCode res = self._PtcWriteDataPoint(self._handle, p, n, radius, cdata) if res==0: raise IOError("Failed to write point cloud data point") + def writeDataPoints(self, numPoints, buffer=None): + """Write a sequence of data points. + + numPoints is the number of points to write. buffer is either a single + buffer that contains all values or a tuple (pointbuf, normalbuf, + radiusbuf, databuf) that each contains the respective value. + The buffers must contain at least numPoints items. + The function accepts ctypes arrays as buffers or any sequence object + that supports the array interface (such as numpy arrays). + """ + # Are there 4 individual buffers? + if type(buffer) is tuple: + if len(buffer)!=4: + raise ValueError("Expected four individual buffers, but got %s"%len(buffer)) + pbuf,nbuf,rbuf,dbuf = buffer + pntPtr = _arrayPointer(pbuf, 3*numPoints) + normPtr = _arrayPointer(nbuf, 3*numPoints) + radPtr = _arrayPointer(rbuf, numPoints) + dataPtr = _arrayPointer(dbuf, self.datasize*numPoints) + pntStride = 3 + normStride = 3 + radStride = 1 + dataStride = self.datasize + # There is only one single buffer for all values + else: + sizeOfFloat = 4 + pntStride = 7+self.datasize + normStride = pntStride + radStride = pntStride + dataStride = pntStride + pntPtr = _arrayPointer(buffer, pntStride*numPoints) + normPtr = pntPtr+3*sizeOfFloat + radPtr = normPtr+3*sizeOfFloat + dataPtr = radPtr+sizeOfFloat + + # Write the points + _pointcloud.writeDataPoints(ctypes.addressof(self._PtcWriteDataPoint), self._handle, numPoints, + pntPtr, pntStride, normPtr, normStride, radPtr, radStride, dataPtr, dataStride) + def _loadPtcLib(self, libName): """Load a RenderMan library providing the point cloud API. """ @@ -306,6 +531,17 @@ def open(fileName, mode="r", libName=None, **kwargs): """Open a point cloud file for reading or writing. + + fileName is the name of the point cloud file. mode is either "r" + for reading a file or "w" for writing a new point cloud file. + libName is the library name that implements the point cloud API. + When mode is "r", the following additional keyword arguments must + be present: + + - vars: A list of tuples (type, name) that defines what additional variables to write + - world2eye: The world2eye matrix + - world2ndc: The world2ndc matrix + - format: A tuple (xres, yres, aspect) """ if mode=="r": return PtcReader(fileName, libName, **kwargs) Added: cgkit/trunk/pyrex/c/_pointcloud.c =================================================================== --- cgkit/trunk/pyrex/c/_pointcloud.c (rev 0) +++ cgkit/trunk/pyrex/c/_pointcloud.c 2009-02-08 14:36:48 UTC (rev 281) @@ -0,0 +1,438 @@ +/* Generated by Pyrex 0.9.8.5 on Sun Feb 8 11:51:47 2009 */ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#include "structmember.h" +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#if PY_VERSION_HEX < 0x02050000 + typedef int Py_ssize_t; + #define PY_SSIZE_T_MAX INT_MAX + #define PY_SSIZE_T_MIN INT_MIN + #define PyInt_FromSsize_t(z) PyInt_FromLong(z) + #define PyInt_AsSsize_t(o) PyInt_AsLong(o) +#endif +#if !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif +#endif +#ifdef __cplusplus +#define __PYX_EXTERN_C extern "C" +#else +#define __PYX_EXTERN_C extern +#endif +#include + + +typedef struct {PyObject **p; int i; char *s; long n;} __Pyx_StringTabEntry; /*proto*/ + +static PyObject *__pyx_m; +static PyObject *__pyx_b; +static int __pyx_lineno; +static char *__pyx_filename; +static char **__pyx_f; + +static char __pyx_mdoc[] = "This module provides helper functions for the pointcloud module.\n"; + +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb); /*proto*/ + +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ + +static void __Pyx_AddTraceback(char *funcname); /*proto*/ + +/* Declarations from _pointcloud */ + + +/* Declarations from implementation of _pointcloud */ + +typedef void *__pyx_t_11_pointcloud_PtcPointCloud; + +typedef int (*__pyx_t_11_pointcloud_PtcReadDataPointPtr)(__pyx_t_11_pointcloud_PtcPointCloud,float *,float *,float *,float *); + +typedef int (*__pyx_t_11_pointcloud_PtcWriteDataPointPtr)(__pyx_t_11_pointcloud_PtcPointCloud,float *,float *,float,float *); + + +static char __pyx_k1[] = "Failed to write data point into point cloud file"; + + +static PyObject *__pyx_k1p; + +static __Pyx_StringTabEntry __pyx_string_tab[] = { + {&__pyx_k1p, 0, __pyx_k1, sizeof(__pyx_k1)}, + {0, 0, 0, 0} +}; + + + +/* Implementation of _pointcloud */ + +static PyObject *__pyx_f_11_pointcloud_readDataPoints(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_11_pointcloud_readDataPoints[] = "Read a sequence of data points.\n \n readFuncLoc is the location of the PtcReadDataPoint function pointer.\n ptcHandle is the open point cloud handle. numPoints is the number of\n points that should be read.\n All the *Buf variables are the respective buffers where the data is written\n to. *Stride is the number of floats to advance the pointers to get to the\n next data location. The buffers must be large enough to hold numPoints\n items.\n The return value is the number of points that were actually read.\n "; +static PyObject *__pyx_f_11_pointcloud_readDataPoints(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + long __pyx_v_readFuncLoc; + long __pyx_v_ptcHandle; + long __pyx_v_numPoints; + long __pyx_v_pointBuf; + int __pyx_v_pointStride; + long __pyx_v_normalBuf; + int __pyx_v_normalStride; + long __pyx_v_radiusBuf; + int __pyx_v_radiusStride; + long __pyx_v_dataBuf; + int __pyx_v_dataStride; + __pyx_t_11_pointcloud_PtcReadDataPointPtr __pyx_v_PtcReadDataPoint; + __pyx_t_11_pointcloud_PtcPointCloud __pyx_v_pointCloud; + float *__pyx_v_pntPtr; + float *__pyx_v_normalPtr; + float *__pyx_v_radiusPtr; + float *__pyx_v_dataPtr; + int __pyx_v_n; + int __pyx_v_res; + PyObject *__pyx_r; + int __pyx_1; + PyObject *__pyx_2 = 0; + static char *__pyx_argnames[] = {"readFuncLoc","ptcHandle","numPoints","pointBuf","pointStride","normalBuf","normalStride","radiusBuf","radiusStride","dataBuf","dataStride",0}; + if (!PyArg_ParseTupleAndKeywords(__pyx_args, __pyx_kwds, "llllililili", __pyx_argnames, &__pyx_v_readFuncLoc, &__pyx_v_ptcHandle, &__pyx_v_numPoints, &__pyx_v_pointBuf, &__pyx_v_pointStride, &__pyx_v_normalBuf, &__pyx_v_normalStride, &__pyx_v_radiusBuf, &__pyx_v_radiusStride, &__pyx_v_dataBuf, &__pyx_v_dataStride)) return 0; + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":61 */ + __pyx_v_PtcReadDataPoint = (((__pyx_t_11_pointcloud_PtcReadDataPointPtr *)__pyx_v_readFuncLoc)[0]); + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":62 */ + __pyx_v_pointCloud = ((__pyx_t_11_pointcloud_PtcPointCloud)__pyx_v_ptcHandle); + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":71 */ + __pyx_v_pntPtr = ((float *)__pyx_v_pointBuf); + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":72 */ + __pyx_v_normalPtr = ((float *)__pyx_v_normalBuf); + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":73 */ + __pyx_v_radiusPtr = ((float *)__pyx_v_radiusBuf); + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":74 */ + __pyx_v_dataPtr = ((float *)__pyx_v_dataBuf); + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":77 */ + __pyx_v_n = 0; + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":78 */ + while (1) { + __pyx_1 = (__pyx_v_n < __pyx_v_numPoints); + if (!__pyx_1) break; + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":79 */ + __pyx_v_res = __pyx_v_PtcReadDataPoint(__pyx_v_pointCloud,__pyx_v_pntPtr,__pyx_v_normalPtr,__pyx_v_radiusPtr,__pyx_v_dataPtr); + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":80 */ + __pyx_1 = (__pyx_v_res == 0); + if (__pyx_1) { + goto __pyx_L3; + goto __pyx_L4; + } + __pyx_L4:; + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":82 */ + __pyx_v_pntPtr += __pyx_v_pointStride; + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":83 */ + __pyx_v_normalPtr += __pyx_v_normalStride; + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":84 */ + __pyx_v_radiusPtr += __pyx_v_radiusStride; + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":85 */ + __pyx_v_dataPtr += __pyx_v_dataStride; + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":86 */ + __pyx_v_n += 1; + } + __pyx_L3:; + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":89 */ + __pyx_2 = PyInt_FromLong(__pyx_v_n); if (!__pyx_2) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; goto __pyx_L1;} + __pyx_r = __pyx_2; + __pyx_2 = 0; + goto __pyx_L0; + + __pyx_r = Py_None; Py_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1:; + Py_XDECREF(__pyx_2); + __Pyx_AddTraceback("_pointcloud.readDataPoints"); + __pyx_r = 0; + __pyx_L0:; + return __pyx_r; +} + +static PyObject *__pyx_f_11_pointcloud_writeDataPoints(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_11_pointcloud_writeDataPoints[] = "Write a sequence of data points.\n \n writeFuncLoc is the location of the PtcWriteDataPoint function pointer.\n ptcHandle is the open point cloud handle. numPoints is the number of\n points that should be written.\n All the *Buf variables are the respective buffers where the data is read\n from. *Stride is the number of floats to advance the pointers to get to the\n next data location. The buffers must contain at least numPoints values.\n "; +static PyObject *__pyx_f_11_pointcloud_writeDataPoints(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + long __pyx_v_writeFuncLoc; + long __pyx_v_ptcHandle; + long __pyx_v_numPoints; + long __pyx_v_pointBuf; + int __pyx_v_pointStride; + long __pyx_v_normalBuf; + int __pyx_v_normalStride; + long __pyx_v_radiusBuf; + int __pyx_v_radiusStride; + long __pyx_v_dataBuf; + int __pyx_v_dataStride; + __pyx_t_11_pointcloud_PtcWriteDataPointPtr __pyx_v_PtcWriteDataPoint; + __pyx_t_11_pointcloud_PtcPointCloud __pyx_v_pointCloud; + float *__pyx_v_pntPtr; + float *__pyx_v_normalPtr; + float *__pyx_v_radiusPtr; + float *__pyx_v_dataPtr; + int __pyx_v_n; + int __pyx_v_res; + PyObject *__pyx_r; + int __pyx_1; + PyObject *__pyx_2 = 0; + PyObject *__pyx_3 = 0; + static char *__pyx_argnames[] = {"writeFuncLoc","ptcHandle","numPoints","pointBuf","pointStride","normalBuf","normalStride","radiusBuf","radiusStride","dataBuf","dataStride",0}; + if (!PyArg_ParseTupleAndKeywords(__pyx_args, __pyx_kwds, "llllililili", __pyx_argnames, &__pyx_v_writeFuncLoc, &__pyx_v_ptcHandle, &__pyx_v_numPoints, &__pyx_v_pointBuf, &__pyx_v_pointStride, &__pyx_v_normalBuf, &__pyx_v_normalStride, &__pyx_v_radiusBuf, &__pyx_v_radiusStride, &__pyx_v_dataBuf, &__pyx_v_dataStride)) return 0; + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":107 */ + __pyx_v_PtcWriteDataPoint = (((__pyx_t_11_pointcloud_PtcWriteDataPointPtr *)__pyx_v_writeFuncLoc)[0]); + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":108 */ + __pyx_v_pointCloud = ((__pyx_t_11_pointcloud_PtcPointCloud)__pyx_v_ptcHandle); + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":117 */ + __pyx_v_pntPtr = ((float *)__pyx_v_pointBuf); + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":118 */ + __pyx_v_normalPtr = ((float *)__pyx_v_normalBuf); + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":119 */ + __pyx_v_radiusPtr = ((float *)__pyx_v_radiusBuf); + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":120 */ + __pyx_v_dataPtr = ((float *)__pyx_v_dataBuf); + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":123 */ + __pyx_v_n = 0; + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":124 */ + while (1) { + __pyx_1 = (__pyx_v_n < __pyx_v_numPoints); + if (!__pyx_1) break; + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":125 */ + __pyx_v_res = __pyx_v_PtcWriteDataPoint(__pyx_v_pointCloud,__pyx_v_pntPtr,__pyx_v_normalPtr,(__pyx_v_radiusPtr[0]),__pyx_v_dataPtr); + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":126 */ + __pyx_1 = (__pyx_v_res == 0); + if (__pyx_1) { + __pyx_2 = PyTuple_New(1); if (!__pyx_2) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 127; goto __pyx_L1;} + Py_INCREF(__pyx_k1p); + PyTuple_SET_ITEM(__pyx_2, 0, __pyx_k1p); + __pyx_3 = PyObject_CallObject(PyExc_IOError, __pyx_2); if (!__pyx_3) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 127; goto __pyx_L1;} + Py_DECREF(__pyx_2); __pyx_2 = 0; + __Pyx_Raise(__pyx_3, 0, 0); + Py_DECREF(__pyx_3); __pyx_3 = 0; + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 127; goto __pyx_L1;} + goto __pyx_L4; + } + __pyx_L4:; + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":128 */ + __pyx_v_pntPtr += __pyx_v_pointStride; + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":129 */ + __pyx_v_normalPtr += __pyx_v_normalStride; + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":130 */ + __pyx_v_radiusPtr += __pyx_v_radiusStride; + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":131 */ + __pyx_v_dataPtr += __pyx_v_dataStride; + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":132 */ + __pyx_v_n += 1; + } + + __pyx_r = Py_None; Py_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1:; + Py_XDECREF(__pyx_2); + Py_XDECREF(__pyx_3); + __Pyx_AddTraceback("_pointcloud.writeDataPoints"); + __pyx_r = 0; + __pyx_L0:; + return __pyx_r; +} + +static struct PyMethodDef __pyx_methods[] = { + {"readDataPoints", (PyCFunction)__pyx_f_11_pointcloud_readDataPoints, METH_VARARGS|METH_KEYWORDS, __pyx_doc_11_pointcloud_readDataPoints}, + {"writeDataPoints", (PyCFunction)__pyx_f_11_pointcloud_writeDataPoints, METH_VARARGS|METH_KEYWORDS, __pyx_doc_11_pointcloud_writeDataPoints}, + {0, 0, 0, 0} +}; + +static void __pyx_init_filenames(void); /*proto*/ + +PyMODINIT_FUNC init_pointcloud(void); /*proto*/ +PyMODINIT_FUNC init_pointcloud(void) { + __pyx_init_filenames(); + __pyx_m = Py_InitModule4("_pointcloud", __pyx_methods, __pyx_mdoc, 0, PYTHON_API_VERSION); + if (!__pyx_m) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 36; goto __pyx_L1;}; + Py_INCREF(__pyx_m); + __pyx_b = PyImport_AddModule("__builtin__"); + if (!__pyx_b) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 36; goto __pyx_L1;}; + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 36; goto __pyx_L1;}; + if (__Pyx_InitStrings(__pyx_string_tab) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 36; goto __pyx_L1;}; + + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":91 */ + return; + __pyx_L1:; + __Pyx_AddTraceback("_pointcloud"); +} + +static char *__pyx_filenames[] = { + "_pointcloud.pyx", +}; + +/* Runtime support code */ + +static void __pyx_init_filenames(void) { + __pyx_f = __pyx_filenames; +} + +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb) { + Py_XINCREF(type); + Py_XINCREF(value); + Py_XINCREF(tb); + /* First, check the traceback argument, replacing None with NULL. */ + if (tb == Py_None) { + Py_DECREF(tb); + tb = 0; + } + else if (tb != NULL && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto raise_error; + } + /* Next, replace a missing value with None */ + if (value == NULL) { + value = Py_None; + Py_INCREF(value); + } + #if PY_VERSION_HEX < 0x02050000 + if (!PyClass_Check(type)) + #else + if (!PyType_Check(type)) + #endif + { + /* Raising an instance. The value should be a dummy. */ + if (value != Py_None) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto raise_error; + } + /* Normalize to raise , */ + Py_DECREF(value); + value = type; + #if PY_VERSION_HEX < 0x02050000 + if (PyInstance_Check(type)) { + type = (PyObject*) ((PyInstanceObject*)type)->in_class; + Py_INCREF(type); + } + else { + PyErr_SetString(PyExc_TypeError, + "raise: exception must be an old-style class or instance"); + goto raise_error; + } + #else + type = (PyObject*) type->ob_type; + Py_INCREF(type); + if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto raise_error; + } + #endif + } + PyErr_Restore(type, value, tb); + return; +raise_error: + Py_XDECREF(value); + Py_XDECREF(type); + Py_XDECREF(tb); + return; +} + +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { + while (t->p) { + *t->p = PyString_FromStringAndSize(t->s, t->n - 1); + if (!*t->p) + return -1; + if (t->i) + PyString_InternInPlace(t->p); + ++t; + } + return 0; +} + +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" + +static void __Pyx_AddTraceback(char *funcname) { + PyObject *py_srcfile = 0; + PyObject *py_funcname = 0; + PyObject *py_globals = 0; + PyObject *empty_tuple = 0; + PyObject *empty_string = 0; + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + + py_srcfile = PyString_FromString(__pyx_filename); + if (!py_srcfile) goto bad; + py_funcname = PyString_FromString(funcname); + if (!py_funcname) goto bad; + py_globals = PyModule_GetDict(__pyx_m); + if (!py_globals) goto bad; + empty_tuple = PyTuple_New(0); + if (!empty_tuple) goto bad; + empty_string = PyString_FromString(""); + if (!empty_string) goto bad; + py_code = PyCode_New( + 0, /*int argcount,*/ + 0, /*int nlocals,*/ + 0, /*int stacksize,*/ + 0, /*int flags,*/ + empty_string, /*PyObject *code,*/ + empty_tuple, /*PyObject *consts,*/ + empty_tuple, /*PyObject *names,*/ + empty_tuple, /*PyObject *varnames,*/ + empty_tuple, /*PyObject *freevars,*/ + empty_tuple, /*PyObject *cellvars,*/ + py_srcfile, /*PyObject *filename,*/ + py_funcname, /*PyObject *name,*/ + __pyx_lineno, /*int firstlineno,*/ + empty_string /*PyObject *lnotab*/ + ); + if (!py_code) goto bad; + py_frame = PyFrame_New( + PyThreadState_Get(), /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + py_globals, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + py_frame->f_lineno = __pyx_lineno; + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_srcfile); + Py_XDECREF(py_funcname); + Py_XDECREF(empty_tuple); + Py_XDECREF(empty_string); + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} Added: cgkit/trunk/pyrex/pyx/_pointcloud.pyx =================================================================== --- cgkit/trunk/pyrex/pyx/_pointcloud.pyx (rev 0) +++ cgkit/trunk/pyrex/pyx/_pointcloud.pyx 2009-02-08 14:36:48 UTC (rev 281) @@ -0,0 +1,132 @@ +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is the Python Computer Graphics Kit. +# +# The Initial Developer of the Original Code is Matthias Baas. +# Portions created by the Initial Developer are Copyright (C) 2009 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** +# $Id: riutil.py,v 1.1.1.1 2004/12/12 14:31:21 mbaas Exp$ + +"""This module provides helper functions for the pointcloud module. +""" + +ctypedef void* PtcPointCloud +ctypedef int (*PtcReadDataPointPtr)(PtcPointCloud, float* pnt, float* normal, float* radius, float* data) +ctypedef int (*PtcWriteDataPointPtr)(PtcPointCloud, float* pnt, float* normal, float radius, float* data) + +def readDataPoints(long readFuncLoc, long ptcHandle, long numPoints, + long pointBuf, int pointStride, + long normalBuf, int normalStride, + long radiusBuf, int radiusStride, + long dataBuf, int dataStride): + """Read a sequence of data points. + + readFuncLoc is the location of the PtcReadDataPoint function pointer. + ptcHandle is the open point cloud handle. numPoints is the number of + points that should be read. + All the *Buf variables are the respective buffers where the data is written + to. *Stride is the number of floats to advance the pointers to get to the + next data location. The buffers must be large enough to hold numPoints + items. + The return value is the number of points that were actually read. + """ + cdef PtcReadDataPointPtr PtcReadDataPoint + cdef PtcPointCloud pointCloud + PtcReadDataPoint = (readFuncLoc)[0] + pointCloud = (ptcHandle) + + cdef float* pntPtr + cdef float* normalPtr + cdef float* radiusPtr + cdef float* dataPtr + cdef int n + cdef int res + + pntPtr = pointBuf + normalPtr = normalBuf + radiusPtr = radiusBuf + dataPtr = dataBuf + + # Read the point data into the given buffers... + n = 0 + while nwriteFuncLoc)[0] + pointCloud = (ptcHandle) + + cdef float* pntPtr + cdef float* normalPtr + cdef float* radiusPtr + cdef float* dataPtr + cdef int n + cdef int res + + pntPtr = pointBuf + normalPtr = normalBuf + radiusPtr = radiusBuf + dataPtr = dataBuf + + # Write the point data into the file... + n = 0 + while n>sys.stderr, "Error running pyrexc" + sys.exit() + +def convertPyxFiles(): + """Convert all pyx files in pyrex/pyx. + """ + pyxFiles = glob.glob("pyrex/pyx/*.pyx") + for pyxFile in pyxFiles: + cFile = os.path.splitext(os.path.basename(pyxFile))[0]+".c" + cFile = os.path.join("pyrex", "c", cFile) + if isNewer(pyxFile, cFile): + pyx2c(pyxFile, cFile) + else: + print "%s is up-to-date"%cFile + ###################################################################### ###################################################################### ###################################################################### @@ -540,7 +564,12 @@ ,extra_link_args=LINK_ARGS ,define_macros=MACROS)] +ext_modules += [Extension("_pointcloud", ["pyrex/c/_pointcloud.c"] + ,extra_compile_args=CC_ARGS + ,extra_link_args=LINK_ARGS + ,define_macros=MACROS)] + # Infos... updateInfoModule(INSTALL_CGKIT_LIGHT) print 70*"=" @@ -594,6 +623,8 @@ if INSTALL_CGKIT_LIGHT: ext_modules = [] +convertPyxFiles() + setup(name = PACKAGE_NAME, version = "2.0.0alpha8cvs", description = "Python Computer Graphics Kit", Added: cgkit/trunk/unittests/test_pointcloud.py =================================================================== --- cgkit/trunk/unittests/test_pointcloud.py (rev 0) +++ cgkit/trunk/unittests/test_pointcloud.py 2009-02-08 14:36:48 UTC (rev 281) @@ -0,0 +1,198 @@ +# Test the pointcloud module + +import unittest +import os, os.path +from cgkit import pointcloud +from cgkit.cgtypes import * +import ctypes +import numpy + +class TestPointCloud(unittest.TestCase): + + def __init__(self, *args, **kwargs): + unittest.TestCase.__init__(self, *args, **kwargs) + self.libName = "3delight" + + def testSinglePoints(self): + """Test writing/reading individual points. + """ + world2eye = mat4(1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16) + world2ndc = mat4(0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.10,0.11,0.12,0.13,0.14,0.15,0.16) + ptc = pointcloud.open("tmp/pointcloud.ptc", "w", self.libName, vars=[], world2eye=world2eye, world2ndc=world2ndc, format=(640,480,1)) + ptc.writeDataPoint((0.1,0.2,0.3), (0,1,0), 0.5, {}) + ptc.writeDataPoint((1,2,3), (1,0,0), 1.0, {}) + ptc.writeDataPoint((-1,0.17,42), (0,0,1), 1.5, {}) + ptc.close() + + ptc = pointcloud.open("tmp/pointcloud.ptc", "r", self.libName) + self.assertEqual(3, ptc.npoints) + self.assertEqual([], ptc.variables) + self.assertEqual(0, ptc.datasize) + print ptc.bbox +# self.assertEqual(16, len(ptc.world2eye)) +# self.assertAlmostEqual(1, ptc.world2eye[0], 5) +# self.assertAlmostEqual(2, ptc.world2eye[1], 5) +# self.assertAlmostEqual(3, ptc.world2eye[2], 5) +# self.assertAlmostEqual(4, ptc.world2eye[3], 5) +# self.assertAlmostEqual(5, ptc.world2eye[4], 5) +# self.assertAlmostEqual(6, ptc.world2eye[5], 5) +# self.assertAlmostEqual(7, ptc.world2eye[6], 5) +# self.assertAlmostEqual(8, ptc.world2eye[7], 5) +# self.assertAlmostEqual(9, ptc.world2eye[8], 5) +# self.assertAlmostEqual(10, ptc.world2eye[9], 5) +# self.assertAlmostEqual(11, ptc.world2eye[10], 5) +# self.assertAlmostEqual(12, ptc.world2eye[11], 5) +# self.assertAlmostEqual(13, ptc.world2eye[12], 5) +# self.assertAlmostEqual(14, ptc.world2eye[13], 5) +# self.assertAlmostEqual(15, ptc.world2eye[14], 5) +# self.assertAlmostEqual(16, ptc.world2eye[15], 5) +# self.assertEqual(16, len(ptc.world2ndc)) +# self.assertAlmostEqual(0.1, ptc.world2ndc[0], 5) +# self.assertAlmostEqual(0.2, ptc.world2ndc[1], 5) +# self.assertAlmostEqual(0.3, ptc.world2ndc[2], 5) +# self.assertAlmostEqual(0.4, ptc.world2ndc[3], 5) +# self.assertAlmostEqual(0.5, ptc.world2ndc[4], 5) +# self.assertAlmostEqual(0.6, ptc.world2ndc[5], 5) +# self.assertAlmostEqual(0.7, ptc.world2ndc[6], 5) +# self.assertAlmostEqual(0.8, ptc.world2ndc[7], 5) +# self.assertAlmostEqual(0.9, ptc.world2ndc[8], 5) +# self.assertAlmostEqual(0.10, ptc.world2ndc[9], 5) +# self.assertAlmostEqual(0.11, ptc.world2ndc[10], 5) +# self.assertAlmostEqual(0.12, ptc.world2ndc[11], 5) +# self.assertAlmostEqual(0.13, ptc.world2ndc[12], 5) +# self.assertAlmostEqual(0.14, ptc.world2ndc[13], 5) +# self.assertAlmostEqual(0.15, ptc.world2ndc[14], 5) +# self.assertAlmostEqual(0.16, ptc.world2ndc[15], 5) + self.assertEqual((640.0, 480.0, 1.0), ptc.format) + + pos,norm,rad,data = ptc.readDataPoint() + self.assertAlmostEqual(0.1, pos[0], 5) + self.assertAlmostEqual(0.2, pos[1], 5) + self.assertAlmostEqual(0.3, pos[2], 5) + self.assertAlmostEqual(0, norm[0], 5) + self.assertAlmostEqual(1, norm[1], 5) + self.assertAlmostEqual(0, norm[2], 5) + self.assertAlmostEqual(0.5, rad, 5) + self.assertEqual({}, data) + + pos,norm,rad,data = ptc.readDataPoint() + self.assertAlmostEqual(1.0, pos[0], 5) + self.assertAlmostEqual(2.0, pos[1], 5) + self.assertAlmostEqual(3.0, pos[2], 5) + self.assertAlmostEqual(1, norm[0], 5) + self.assertAlmostEqual(0, norm[1], 5) + self.assertAlmostEqual(0, norm[2], 5) + self.assertAlmostEqual(1.0, rad, 5) + self.assertEqual({}, data) + + pos,norm,rad,data = ptc.readDataPoint() + self.assertAlmostEqual(-1.0, pos[0], 5) + self.assertAlmostEqual(0.17, pos[1], 5) + self.assertAlmostEqual(42.0, pos[2], 5) + self.assertAlmostEqual(0, norm[0], 5) + self.assertAlmostEqual(0, norm[1], 5) + self.assertAlmostEqual(1, norm[2], 5) + self.assertAlmostEqual(1.5, rad, 5) + self.assertEqual({}, data) + + v = ptc.readDataPoint() + self.assertEqual(None, v) + + ptc.close() + + def testMultiPoints(self): + """Test writing/reading several points at once. + """ + pnts = (6*ctypes.c_float)(0.4, 0.8, 1.0, 0.9, 0.7, 0.6) + norms = (6*ctypes.c_float)(1,0,0, 0,0,1) + rads = (2*ctypes.c_float)(0.4, 0.5) + data = (8*ctypes.c_float)(12.0, 1,2,3, 42.0, -1,-2,-3) + ptc = pointcloud.open("tmp/pointcloud2.ptc", "w", self.libName, vars=[("float", "fspam"), ("vector", "vspam")], world2eye=mat4(1), world2ndc=mat4(1), format=(640,480,1)) + ptc.writeDataPoints(2, (pnts,norms,rads,data)) + ptc.close() + + ptc = pointcloud.open("tmp/pointcloud2.ptc", "r", self.libName) + self.assertEqual(2, ptc.npoints) + self.assertEqual([("float","fspam"), ("vector","vspam")], ptc.variables) + self.assertEqual(4, ptc.datasize) + ps = (6*ctypes.c_float)() + ns = (6*ctypes.c_float)() + rs = (2*ctypes.c_float)() + ds = (8*ctypes.c_float)() + ptc.readDataPoints(2, (ps,ns,rs,ds)) + self.assertAlmostEqual(0.4, ps[0], 5) + self.assertAlmostEqual(0.8, ps[1], 5) + self.assertAlmostEqual(1.0, ps[2], 5) + self.assertAlmostEqual(0.9, ps[3], 5) + self.assertAlmostEqual(0.7, ps[4], 5) + self.assertAlmostEqual(0.6, ps[5], 5) + self.assertAlmostEqual(1, ns[0], 5) + self.assertAlmostEqual(0, ns[1], 5) + self.assertAlmostEqual(0, ns[2], 5) + self.assertAlmostEqual(0, ns[3], 5) + self.assertAlmostEqual(0, ns[4], 5) + self.assertAlmostEqual(1, ns[5], 5) + self.assertAlmostEqual(0.4, rs[0], 5) + self.assertAlmostEqual(0.5, rs[1], 5) + self.assertAlmostEqual(12.0, ds[0], 5) + self.assertAlmostEqual(1, ds[1], 5) + self.assertAlmostEqual(2, ds[2], 5) + self.assertAlmostEqual(3, ds[3], 5) + self.assertAlmostEqual(42.0, ds[4], 5) + self.assertAlmostEqual(-1, ds[5], 5) + self.assertAlmostEqual(-2, ds[6], 5) + self.assertAlmostEqual(-3, ds[7], 5) + ptc.close() + + ptc = pointcloud.open("tmp/pointcloud2.ptc", "r", self.libName) + pos,norm,rad,data = ptc.readDataPoint() + self.assertAlmostEqual(0.4, pos[0], 5) + self.assertAlmostEqual(0.8, pos[1], 5) + self.assertAlmostEqual(1.0, pos[2], 5) + self.assertAlmostEqual(1, norm[0], 5) + self.assertAlmostEqual(0, norm[1], 5) + self.assertAlmostEqual(0, norm[2], 5) + self.assertAlmostEqual(0.4, rad, 5) + self.assertAlmostEqual(12, data["fspam"], 5) + self.assertAlmostEqual(1, data["vspam"][0], 5) + self.assertAlmostEqual(2, data["vspam"][1], 5) + self.assertAlmostEqual(3, data["vspam"][2], 5) + ptc.close() + + ptc = pointcloud.open("tmp/pointcloud2.ptc", "r", self.libName) + for buf in ptc.iterBatches(10, numpyArray=False, combinedBuffer=True): + self.assertEqual(22, len(buf)) + self.assertAlmostEqual(0.4, buf[0], 5) + self.assertAlmostEqual(0.8, buf[1], 5) + self.assertAlmostEqual(1.0, buf[2], 5) + self.assertAlmostEqual(1, buf[3], 5) + self.assertAlmostEqual(0, buf[4], 5) + self.assertAlmostEqual(0, buf[5], 5) + self.assertAlmostEqual(0.4, buf[6], 5) + self.assertAlmostEqual(12, buf[7], 5) + self.assertAlmostEqual(1, buf[8], 5) + self.assertAlmostEqual(2, buf[9], 5) + self.assertAlmostEqual(3, buf[10], 5) + ptc.close() + + def testMultiPointsOneBuffer(self): + """Test writing/reading several points at once. + """ + buffer = numpy.zeros(shape=(2,8), dtype=numpy.float32) + buffer[0] = (1,2,3,1,0,0,7,8) + buffer[1] = (-1,-2,-3,0,0,1,-7,-8) + ptc = pointcloud.open("tmp/pointcloud3.ptc", "w", self.libName, vars=[("float", "spam")], world2eye=mat4(1), world2ndc=mat4(1), format=(640,480,1)) + ptc.writeDataPoints(2, buffer) + ptc.close() + + buf = numpy.zeros(shape=(2,8), dtype=numpy.float32) + ptc = pointcloud.open("tmp/pointcloud3.ptc", "r", self.libName) + ptc.readDataPoints(2, buf) + self.assertEqual([1,2,3,1,0,0,7,8], list(list(buf)[0])) + self.assertEqual([-1,-2,-3,0,0,1,-7,-8], list(list(buf)[1])) + ptc.close() + +###################################################################### + +if __name__=="__main__": + unittest.main() This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. 
 [cgkit-commits] SF.net SVN: cgkit:[285] cgkit/trunk From: - 2009-02-10 07:49:45 Revision: 285 http://cgkit.svn.sourceforge.net/cgkit/?rev=285&view=rev Author: mbaas Date: 2009-02-10 07:49:42 +0000 (Tue, 10 Feb 2009) Log Message: ----------- The point cloud reader now tracks the number of points that have been read. Modified Paths: -------------- cgkit/trunk/cgkit/pointcloud.py cgkit/trunk/pyrex/c/_pointcloud.c cgkit/trunk/pyrex/pyx/_pointcloud.pyx cgkit/trunk/unittests/test_pointcloud.py Modified: cgkit/trunk/cgkit/pointcloud.py =================================================================== --- cgkit/trunk/cgkit/pointcloud.py 2009-02-09 22:04:25 UTC (rev 284) +++ cgkit/trunk/cgkit/pointcloud.py 2009-02-10 07:49:42 UTC (rev 285) @@ -128,6 +128,7 @@ self._handle = handle + self.name = fileName self.variables = [] self.npoints = None self.bbox = None @@ -175,9 +176,16 @@ if ptclib.PtcGetPointCloudInfo(handle, "world2ndc", m)==1: self.world2ndc = list(m) + if self.npoints is None: + raise IOError("Could not obtain the number of points in point cloud file %s."%fileName) + if self.npoints<0: + raise ValueError("The number of points in the point cloud file is negative (%s)."%self.npoints) if self.datasize is None: raise IOError("Could not obtain datasize value from point cloud file %s."%fileName) + # The number of points that can still be read before eof is hit + self._numPointsLeft = self.npoints + # Set up storage for reading individual data points self._pos = (3*ctypes.c_float)() self._normal = (3*ctypes.c_float)() @@ -206,14 +214,19 @@ Returns a tuple (pos, normal, radius, dataDict) where pos and normal are 3-tuples of floats, radius is a single float and dataDict a dictionary with the extra variables that are attached to the point. - If no more point is available None is returned. + If no more point is available an EOFError exception is thrown. + An IOErrror handle is thrown when an error occurs during reading or + when the file has already been closed. """ if self._handle is None: - raise IOError("The point cloud file has already been closed.") + raise IOError("The point cloud file has already been closed (%s)"%self.name) + if self._numPointsLeft==0: + raise EOFError("There are no more points left to read from point cloud file %s"%self.name) + self._numPointsLeft -= 1 res = self._PtcReadDataPoint(self._handle, self._pos, self._normal, self._radius, self._data) if res==0: - return None + raise IOError("Error while reading data point from point cloud file %s"%self.name) else: dataDict = {} data = self._data @@ -233,8 +246,11 @@ The return value is the number of points that have actually been read (additional items in the buffers remain at their previous - value). + value). When 0 is returned, the end of the file has been reached. """ + if numPoints<=0: + return 0 + # Are there 4 individual buffers? if type(buffer) is tuple: if len(buffer)!=4: @@ -260,10 +276,13 @@ radPtr = normPtr+3*sizeOfFloat dataPtr = radPtr+sizeOfFloat + num = min(numPoints, self._numPointsLeft) + # Read the points - n = _pointcloud.readDataPoints(ctypes.addressof(self._PtcReadDataPoint), self._handle, numPoints, - pntPtr, pntStride, normPtr, normStride, radPtr, radStride, dataPtr, dataStride) - return n + self._numPointsLeft -= num + _pointcloud.readDataPoints(ctypes.addressof(self._PtcReadDataPoint), self._handle, num, + pntPtr, pntStride, normPtr, normStride, radPtr, radStride, dataPtr, dataStride) + return num def iterPoints(self): """Iterate over all the points in the file. @@ -271,11 +290,8 @@ Yields tuples (point,normal,radius,data) for every point in the file. This is equivalent to calling readDataPoint() repeatedly. """ - while 1: - data = self.readDataPoint() - if data is None: - break - yield data + while self._numPointsLeft>0: + yield self.readDataPoint() def iterBatches(self, batchSize=1000, combinedBuffer=False, numpyArray=False): """Iterate over point batches. @@ -295,7 +311,7 @@ if numpyArray and not _numpy_available: raise ImportError("numpy is not available") - num = self.npoints + num = self._numPointsLeft buffer = None bufLen = 0 while num>0: Modified: cgkit/trunk/pyrex/c/_pointcloud.c =================================================================== --- cgkit/trunk/pyrex/c/_pointcloud.c 2009-02-09 22:04:25 UTC (rev 284) +++ cgkit/trunk/pyrex/c/_pointcloud.c 2009-02-10 07:49:42 UTC (rev 285) @@ -1,4 +1,4 @@ -/* Generated by Pyrex 0.9.8.5 on Sun Feb 8 11:51:47 2009 */ +/* Generated by Pyrex 0.9.8.5 on Tue Feb 10 07:47:20 2009 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -57,13 +57,16 @@ typedef int (*__pyx_t_11_pointcloud_PtcWriteDataPointPtr)(__pyx_t_11_pointcloud_PtcPointCloud,float *,float *,float,float *); -static char __pyx_k1[] = "Failed to write data point into point cloud file"; +static char __pyx_k1[] = "Failed to read data point from point cloud file"; +static char __pyx_k2[] = "Failed to write data point into point cloud file"; static PyObject *__pyx_k1p; +static PyObject *__pyx_k2p; static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_k1p, 0, __pyx_k1, sizeof(__pyx_k1)}, + {&__pyx_k2p, 0, __pyx_k2, sizeof(__pyx_k2)}, {0, 0, 0, 0} }; @@ -72,7 +75,7 @@ /* Implementation of _pointcloud */ static PyObject *__pyx_f_11_pointcloud_readDataPoints(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_11_pointcloud_readDataPoints[] = "Read a sequence of data points.\n \n readFuncLoc is the location of the PtcReadDataPoint function pointer.\n ptcHandle is the open point cloud handle. numPoints is the number of\n points that should be read.\n All the *Buf variables are the respective buffers where the data is written\n to. *Stride is the number of floats to advance the pointers to get to the\n next data location. The buffers must be large enough to hold numPoints\n items.\n The return value is the number of points that were actually read.\n "; +static char __pyx_doc_11_pointcloud_readDataPoints[] = "Read a sequence of data points.\n \n readFuncLoc is the location of the PtcReadDataPoint function pointer.\n ptcHandle is the open point cloud handle. numPoints is the number of\n points that should be read.\n All the *Buf variables are the respective buffers where the data is written\n to. *Stride is the number of floats to advance the pointers to get to the\n next data location. The buffers must be large enough to hold numPoints\n items.\n numPoints must at least be the number of points that are still left\n in the file (otherwise the function may generate an exception or return\n bogus values, whatever the underlying RenderMan implementation does).\n "; static PyObject *__pyx_f_11_pointcloud_readDataPoints(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { long __pyx_v_readFuncLoc; long __pyx_v_ptcHandle; @@ -96,73 +99,75 @@ PyObject *__pyx_r; int __pyx_1; PyObject *__pyx_2 = 0; + PyObject *__pyx_3 = 0; static char *__pyx_argnames[] = {"readFuncLoc","ptcHandle","numPoints","pointBuf","pointStride","normalBuf","normalStride","radiusBuf","radiusStride","dataBuf","dataStride",0}; if (!PyArg_ParseTupleAndKeywords(__pyx_args, __pyx_kwds, "llllililili", __pyx_argnames, &__pyx_v_readFuncLoc, &__pyx_v_ptcHandle, &__pyx_v_numPoints, &__pyx_v_pointBuf, &__pyx_v_pointStride, &__pyx_v_normalBuf, &__pyx_v_normalStride, &__pyx_v_radiusBuf, &__pyx_v_radiusStride, &__pyx_v_dataBuf, &__pyx_v_dataStride)) return 0; - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":61 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":63 */ __pyx_v_PtcReadDataPoint = (((__pyx_t_11_pointcloud_PtcReadDataPointPtr *)__pyx_v_readFuncLoc)[0]); - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":62 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":64 */ __pyx_v_pointCloud = ((__pyx_t_11_pointcloud_PtcPointCloud)__pyx_v_ptcHandle); - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":71 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":73 */ __pyx_v_pntPtr = ((float *)__pyx_v_pointBuf); - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":72 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":74 */ __pyx_v_normalPtr = ((float *)__pyx_v_normalBuf); - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":73 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":75 */ __pyx_v_radiusPtr = ((float *)__pyx_v_radiusBuf); - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":74 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":76 */ __pyx_v_dataPtr = ((float *)__pyx_v_dataBuf); - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":77 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":79 */ __pyx_v_n = 0; - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":78 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":80 */ while (1) { __pyx_1 = (__pyx_v_n < __pyx_v_numPoints); if (!__pyx_1) break; - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":79 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":81 */ __pyx_v_res = __pyx_v_PtcReadDataPoint(__pyx_v_pointCloud,__pyx_v_pntPtr,__pyx_v_normalPtr,__pyx_v_radiusPtr,__pyx_v_dataPtr); - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":80 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":82 */ __pyx_1 = (__pyx_v_res == 0); if (__pyx_1) { - goto __pyx_L3; + __pyx_2 = PyTuple_New(1); if (!__pyx_2) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; goto __pyx_L1;} + Py_INCREF(__pyx_k1p); + PyTuple_SET_ITEM(__pyx_2, 0, __pyx_k1p); + __pyx_3 = PyObject_CallObject(PyExc_IOError, __pyx_2); if (!__pyx_3) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; goto __pyx_L1;} + Py_DECREF(__pyx_2); __pyx_2 = 0; + __Pyx_Raise(__pyx_3, 0, 0); + Py_DECREF(__pyx_3); __pyx_3 = 0; + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 83; goto __pyx_L1;} goto __pyx_L4; } __pyx_L4:; - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":82 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":84 */ __pyx_v_pntPtr += __pyx_v_pointStride; - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":83 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":85 */ __pyx_v_normalPtr += __pyx_v_normalStride; - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":84 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":86 */ __pyx_v_radiusPtr += __pyx_v_radiusStride; - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":85 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":87 */ __pyx_v_dataPtr += __pyx_v_dataStride; - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":86 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":88 */ __pyx_v_n += 1; } - __pyx_L3:; - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":89 */ - __pyx_2 = PyInt_FromLong(__pyx_v_n); if (!__pyx_2) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; goto __pyx_L1;} - __pyx_r = __pyx_2; - __pyx_2 = 0; - goto __pyx_L0; - __pyx_r = Py_None; Py_INCREF(Py_None); goto __pyx_L0; __pyx_L1:; Py_XDECREF(__pyx_2); + Py_XDECREF(__pyx_3); __Pyx_AddTraceback("_pointcloud.readDataPoints"); __pyx_r = 0; __pyx_L0:; @@ -198,63 +203,63 @@ static char *__pyx_argnames[] = {"writeFuncLoc","ptcHandle","numPoints","pointBuf","pointStride","normalBuf","normalStride","radiusBuf","radiusStride","dataBuf","dataStride",0}; if (!PyArg_ParseTupleAndKeywords(__pyx_args, __pyx_kwds, "llllililili", __pyx_argnames, &__pyx_v_writeFuncLoc, &__pyx_v_ptcHandle, &__pyx_v_numPoints, &__pyx_v_pointBuf, &__pyx_v_pointStride, &__pyx_v_normalBuf, &__pyx_v_normalStride, &__pyx_v_radiusBuf, &__pyx_v_radiusStride, &__pyx_v_dataBuf, &__pyx_v_dataStride)) return 0; - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":107 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":106 */ __pyx_v_PtcWriteDataPoint = (((__pyx_t_11_pointcloud_PtcWriteDataPointPtr *)__pyx_v_writeFuncLoc)[0]); - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":108 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":107 */ __pyx_v_pointCloud = ((__pyx_t_11_pointcloud_PtcPointCloud)__pyx_v_ptcHandle); - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":117 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":116 */ __pyx_v_pntPtr = ((float *)__pyx_v_pointBuf); - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":118 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":117 */ __pyx_v_normalPtr = ((float *)__pyx_v_normalBuf); - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":119 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":118 */ __pyx_v_radiusPtr = ((float *)__pyx_v_radiusBuf); - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":120 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":119 */ __pyx_v_dataPtr = ((float *)__pyx_v_dataBuf); - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":123 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":122 */ __pyx_v_n = 0; - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":124 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":123 */ while (1) { __pyx_1 = (__pyx_v_n < __pyx_v_numPoints); if (!__pyx_1) break; - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":125 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":124 */ __pyx_v_res = __pyx_v_PtcWriteDataPoint(__pyx_v_pointCloud,__pyx_v_pntPtr,__pyx_v_normalPtr,(__pyx_v_radiusPtr[0]),__pyx_v_dataPtr); - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":126 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":125 */ __pyx_1 = (__pyx_v_res == 0); if (__pyx_1) { - __pyx_2 = PyTuple_New(1); if (!__pyx_2) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 127; goto __pyx_L1;} - Py_INCREF(__pyx_k1p); - PyTuple_SET_ITEM(__pyx_2, 0, __pyx_k1p); - __pyx_3 = PyObject_CallObject(PyExc_IOError, __pyx_2); if (!__pyx_3) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 127; goto __pyx_L1;} + __pyx_2 = PyTuple_New(1); if (!__pyx_2) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; goto __pyx_L1;} + Py_INCREF(__pyx_k2p); + PyTuple_SET_ITEM(__pyx_2, 0, __pyx_k2p); + __pyx_3 = PyObject_CallObject(PyExc_IOError, __pyx_2); if (!__pyx_3) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; goto __pyx_L1;} Py_DECREF(__pyx_2); __pyx_2 = 0; __Pyx_Raise(__pyx_3, 0, 0); Py_DECREF(__pyx_3); __pyx_3 = 0; - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 127; goto __pyx_L1;} + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 126; goto __pyx_L1;} goto __pyx_L4; } __pyx_L4:; - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":128 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":127 */ __pyx_v_pntPtr += __pyx_v_pointStride; - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":129 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":128 */ __pyx_v_normalPtr += __pyx_v_normalStride; - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":130 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":129 */ __pyx_v_radiusPtr += __pyx_v_radiusStride; - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":131 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":130 */ __pyx_v_dataPtr += __pyx_v_dataStride; - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":132 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":131 */ __pyx_v_n += 1; } @@ -288,7 +293,7 @@ if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 36; goto __pyx_L1;}; if (__Pyx_InitStrings(__pyx_string_tab) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 36; goto __pyx_L1;}; - /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":91 */ + /* "/Users/baas/src/cgkit/pyrex/pyx/_pointcloud.pyx":90 */ return; __pyx_L1:; __Pyx_AddTraceback("_pointcloud"); Modified: cgkit/trunk/pyrex/pyx/_pointcloud.pyx =================================================================== --- cgkit/trunk/pyrex/pyx/_pointcloud.pyx 2009-02-09 22:04:25 UTC (rev 284) +++ cgkit/trunk/pyrex/pyx/_pointcloud.pyx 2009-02-10 07:49:42 UTC (rev 285) @@ -54,7 +54,9 @@ to. *Stride is the number of floats to advance the pointers to get to the next data location. The buffers must be large enough to hold numPoints items. - The return value is the number of points that were actually read. + numPoints must at least be the number of points that are still left + in the file (otherwise the function may generate an exception or return + bogus values, whatever the underlying RenderMan implementation does). """ cdef PtcReadDataPointPtr PtcReadDataPoint cdef PtcPointCloud pointCloud @@ -78,15 +80,12 @@ while n
 [cgkit-commits] SF.net SVN: cgkit:[288] cgkit/trunk From: - 2009-02-10 22:48:23 Revision: 288 http://cgkit.svn.sourceforge.net/cgkit/?rev=288&view=rev Author: mbaas Date: 2009-02-10 22:43:44 +0000 (Tue, 10 Feb 2009) Log Message: ----------- Support mat4 values. Modified Paths: -------------- cgkit/trunk/cgkit/pointcloud.py cgkit/trunk/unittests/test_pointcloud.py Modified: cgkit/trunk/cgkit/pointcloud.py =================================================================== --- cgkit/trunk/cgkit/pointcloud.py 2009-02-10 09:33:25 UTC (rev 287) +++ cgkit/trunk/cgkit/pointcloud.py 2009-02-10 22:43:44 UTC (rev 288) @@ -36,6 +36,7 @@ import os.path import ctypes +import cgtypes import rmanlibutil import _pointcloud try: @@ -434,14 +435,17 @@ m can be any object that contains 16 floats (the values may be nested). """ - values = [] - for v in m: - try: - values.extend(list(v)) - except: - values.append(v) - if len(values)!=16: - raise ValueError("Matrix must be composed of 16 values, got %s instead."%len(values)) + if isinstance(m, cgtypes.mat4): + values = m.toList(rowmajor=True) + else: + values = [] + for v in m: + try: + values.extend(list(v)) + except: + values.append(v) + if len(values)!=16: + raise ValueError("Matrix must be composed of 16 values, got %s instead."%len(values)) return (16*ctypes.c_float)(*values) def __del__(self): Modified: cgkit/trunk/unittests/test_pointcloud.py =================================================================== --- cgkit/trunk/unittests/test_pointcloud.py 2009-02-10 09:33:25 UTC (rev 287) +++ cgkit/trunk/unittests/test_pointcloud.py 2009-02-10 22:43:44 UTC (rev 288) @@ -39,40 +39,40 @@ self.assertAlmostEqual( 1.0, ptc.bbox[3], 1) self.assertAlmostEqual( 2.0, ptc.bbox[4], 1) self.assertAlmostEqual(42.0, ptc.bbox[5], 1) -# self.assertEqual(16, len(ptc.world2eye)) -# self.assertAlmostEqual(1, ptc.world2eye[0], self.accuracy) -# self.assertAlmostEqual(2, ptc.world2eye[1], self.accuracy) -# self.assertAlmostEqual(3, ptc.world2eye[2], self.accuracy) -# self.assertAlmostEqual(4, ptc.world2eye[3], self.accuracy) -# self.assertAlmostEqual(5, ptc.world2eye[4], self.accuracy) -# self.assertAlmostEqual(6, ptc.world2eye[5], self.accuracy) -# self.assertAlmostEqual(7, ptc.world2eye[6], self.accuracy) -# self.assertAlmostEqual(8, ptc.world2eye[7], self.accuracy) -# self.assertAlmostEqual(9, ptc.world2eye[8], self.accuracy) -# self.assertAlmostEqual(10, ptc.world2eye[9], self.accuracy) -# self.assertAlmostEqual(11, ptc.world2eye[10], self.accuracy) -# self.assertAlmostEqual(12, ptc.world2eye[11], self.accuracy) -# self.assertAlmostEqual(13, ptc.world2eye[12], self.accuracy) -# self.assertAlmostEqual(14, ptc.world2eye[13], self.accuracy) -# self.assertAlmostEqual(15, ptc.world2eye[14], self.accuracy) -# self.assertAlmostEqual(16, ptc.world2eye[15], self.accuracy) -# self.assertEqual(16, len(ptc.world2ndc)) -# self.assertAlmostEqual(0.1, ptc.world2ndc[0], self.accuracy) -# self.assertAlmostEqual(0.2, ptc.world2ndc[1], self.accuracy) -# self.assertAlmostEqual(0.3, ptc.world2ndc[2], self.accuracy) -# self.assertAlmostEqual(0.4, ptc.world2ndc[3], self.accuracy) -# self.assertAlmostEqual(0.5, ptc.world2ndc[4], self.accuracy) -# self.assertAlmostEqual(0.6, ptc.world2ndc[5], self.accuracy) -# self.assertAlmostEqual(0.7, ptc.world2ndc[6], self.accuracy) -# self.assertAlmostEqual(0.8, ptc.world2ndc[7], self.accuracy) -# self.assertAlmostEqual(0.9, ptc.world2ndc[8], self.accuracy) -# self.assertAlmostEqual(0.10, ptc.world2ndc[9], self.accuracy) -# self.assertAlmostEqual(0.11, ptc.world2ndc[10], self.accuracy) -# self.assertAlmostEqual(0.12, ptc.world2ndc[11], self.accuracy) -# self.assertAlmostEqual(0.13, ptc.world2ndc[12], self.accuracy) -# self.assertAlmostEqual(0.14, ptc.world2ndc[13], self.accuracy) -# self.assertAlmostEqual(0.15, ptc.world2ndc[14], self.accuracy) -# self.assertAlmostEqual(0.16, ptc.world2ndc[15], self.accuracy) + self.assertEqual(16, len(ptc.world2eye)) + self.assertAlmostEqual(1, ptc.world2eye[0], self.accuracy) + self.assertAlmostEqual(2, ptc.world2eye[1], self.accuracy) + self.assertAlmostEqual(3, ptc.world2eye[2], self.accuracy) + self.assertAlmostEqual(4, ptc.world2eye[3], self.accuracy) + self.assertAlmostEqual(5, ptc.world2eye[4], self.accuracy) + self.assertAlmostEqual(6, ptc.world2eye[5], self.accuracy) + self.assertAlmostEqual(7, ptc.world2eye[6], self.accuracy) + self.assertAlmostEqual(8, ptc.world2eye[7], self.accuracy) + self.assertAlmostEqual(9, ptc.world2eye[8], self.accuracy) + self.assertAlmostEqual(10, ptc.world2eye[9], self.accuracy) + self.assertAlmostEqual(11, ptc.world2eye[10], self.accuracy) + self.assertAlmostEqual(12, ptc.world2eye[11], self.accuracy) + self.assertAlmostEqual(13, ptc.world2eye[12], self.accuracy) + self.assertAlmostEqual(14, ptc.world2eye[13], self.accuracy) + self.assertAlmostEqual(15, ptc.world2eye[14], self.accuracy) + self.assertAlmostEqual(16, ptc.world2eye[15], self.accuracy) + self.assertEqual(16, len(ptc.world2ndc)) + self.assertAlmostEqual(0.1, ptc.world2ndc[0], self.accuracy) + self.assertAlmostEqual(0.2, ptc.world2ndc[1], self.accuracy) + self.assertAlmostEqual(0.3, ptc.world2ndc[2], self.accuracy) + self.assertAlmostEqual(0.4, ptc.world2ndc[3], self.accuracy) + self.assertAlmostEqual(0.5, ptc.world2ndc[4], self.accuracy) + self.assertAlmostEqual(0.6, ptc.world2ndc[5], self.accuracy) + self.assertAlmostEqual(0.7, ptc.world2ndc[6], self.accuracy) + self.assertAlmostEqual(0.8, ptc.world2ndc[7], self.accuracy) + self.assertAlmostEqual(0.9, ptc.world2ndc[8], self.accuracy) + self.assertAlmostEqual(0.10, ptc.world2ndc[9], self.accuracy) + self.assertAlmostEqual(0.11, ptc.world2ndc[10], self.accuracy) + self.assertAlmostEqual(0.12, ptc.world2ndc[11], self.accuracy) + self.assertAlmostEqual(0.13, ptc.world2ndc[12], self.accuracy) + self.assertAlmostEqual(0.14, ptc.world2ndc[13], self.accuracy) + self.assertAlmostEqual(0.15, ptc.world2ndc[14], self.accuracy) + self.assertAlmostEqual(0.16, ptc.world2ndc[15], self.accuracy) self.assertEqual((640.0, 480.0, 1.0), ptc.format) pos,norm,rad,data = ptc.readDataPoint() This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. 
 [cgkit-commits] SF.net SVN: cgkit:[296] cgkit/trunk From: - 2009-03-01 11:09:20 Revision: 296 http://cgkit.svn.sourceforge.net/cgkit/?rev=296&view=rev Author: mbaas Date: 2009-03-01 11:09:15 +0000 (Sun, 01 Mar 2009) Log Message: ----------- Fixed some typos Modified Paths: -------------- cgkit/trunk/cgkit/pointcloud.py cgkit/trunk/doc/tex/pointcloud.tex Modified: cgkit/trunk/cgkit/pointcloud.py =================================================================== --- cgkit/trunk/cgkit/pointcloud.py 2009-03-01 11:08:26 UTC (rev 295) +++ cgkit/trunk/cgkit/pointcloud.py 2009-03-01 11:09:15 UTC (rev 296) @@ -266,7 +266,7 @@ are 3-tuples of floats, radius is a single float and dataDict a dictionary with the extra variables that are attached to the point. If no more point is available an EOFError exception is thrown. - An IOErrror handle is thrown when an error occurs during reading or + An IOErrror exception is thrown when an error occurs during reading or when the file has already been closed. """ if self._handle is None: @@ -284,7 +284,7 @@ exec self._dataCollectionCode return tuple(self._pos), tuple(self._normal), self._radius.value, dataDict - def readDataPoints(self, numPoints, buffer=None): + def readDataPoints(self, numPoints, buffer): """Read a sequence of data points. numPoints is the number of points to read. buffer is either a single @@ -347,10 +347,10 @@ def iterBatches(self, batchSize=1000, combinedBuffer=False, numpyArray=False): """Iterate over point batches. - Reads batchSize points at once and yields one or more buffer + Reads batchSize points at once and yields one or more buffers containing the data. combinedBuffer determines whether all data is written into one single - buffer or if there is an individual buffer for the point, norma, radius + buffer or if there is an individual buffer for the point, normal, radius and data. numpyArray determines whether the buffers are created as numpy arrays or ctypes arrays. @@ -445,6 +445,8 @@ self._PtcWriteDataPoint = ptclib.PtcWriteDataPoint self._PtcFinishPointCloudFile = ptclib.PtcFinishPointCloudFile + self.name = fileName + xres,yres,aspect = format w2e = self._matrixToCTypes(world2eye) @@ -518,7 +520,7 @@ point and normal are vectors (any 3-sequence of floats) and radius a float. data is a dict that contains the extra variables that must have been declared in the constructor. Undeclared values are - ignored, missing declared valued are set to 0. + ignored, missing declared values are set to 0. """ if self._handle is None: raise IOError("The point cloud file has already been closed.") @@ -531,7 +533,7 @@ if res==0: raise IOError("Failed to write point cloud data point") - def writeDataPoints(self, numPoints, buffer=None): + def writeDataPoints(self, numPoints, buffer): """Write a sequence of data points. numPoints is the number of points to write. buffer is either a single @@ -607,7 +609,7 @@ fileName is the name of the point cloud file. mode is either "r" for reading a file or "w" for writing a new point cloud file. libName is the library name that implements the point cloud API. - When mode is "r", the following additional keyword arguments must + When mode is "w", the following additional keyword arguments must be present: - vars: A list of tuples (type, name) that defines what additional variables to write Modified: cgkit/trunk/doc/tex/pointcloud.tex =================================================================== --- cgkit/trunk/doc/tex/pointcloud.tex 2009-03-01 11:08:26 UTC (rev 295) +++ cgkit/trunk/doc/tex/pointcloud.tex 2009-03-01 11:09:15 UTC (rev 296) @@ -109,7 +109,7 @@ an entire sequence of points at once. \end{methoddesc} -\begin{methoddesc}{readDataPoints}{numPoints, buffer=None} +\begin{methoddesc}{readDataPoints}{numPoints, buffer} Read a sequence of data points. \var{numPoints} is the number of points to read. \var{buffer} is either a single buffer that will receive all values or a tuple (\var{pointbuf}, \var{normalbuf}, This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. 
 [cgkit-commits] SF.net SVN: cgkit:[297] cgkit/trunk From: - 2009-03-01 11:10:51 Revision: 297 http://cgkit.svn.sourceforge.net/cgkit/?rev=297&view=rev Author: mbaas Date: 2009-03-01 11:10:48 +0000 (Sun, 01 Mar 2009) Log Message: ----------- Added RenderMan calls that are newer than the 3.2 spec. Modified Paths: -------------- cgkit/trunk/cgkit/_cri.py cgkit/trunk/cgkit/cri.py cgkit/trunk/cgkit/ri.py cgkit/trunk/doc/tex/ri.tex Modified: cgkit/trunk/cgkit/_cri.py =================================================================== --- cgkit/trunk/cgkit/_cri.py 2009-03-01 11:09:15 UTC (rev 296) +++ cgkit/trunk/cgkit/_cri.py 2009-03-01 11:10:48 UTC (rev 297) @@ -455,7 +455,87 @@ ri.RiSincFilter.argtypes = [c_float, c_float, c_float, c_float] ri.RiSincFilter.restype = c_float + # Optional calls from the 3.4 API: + if hasattr(ri, "RiArchiveBegin"): + ri.RiArchiveBegin.argtypes = [RtToken] + ri.RiArchiveBegin.restype = c_void_p + else: + ri.RiArchiveBegin = _undefinedFunctionSubstitute + + if hasattr(ri, "RiArchiveEnd"): + ri.RiArchiveEnd.argtypes = [] + else: + ri.RiArchiveEnd = _undefinedFunctionSubstitute + + if hasattr(ri, "RiCamera"): + ri.RiCamera.argtypes = [RtToken] + else: + ri.RiCamera = _undefinedFunctionSubstitute + + if hasattr(ri, "RiDisplayChannel"): + ri.RiDisplayChannel.argtypes = [RtToken] + else: + ri.RiDisplayChannel = _undefinedFunctionSubstitute + + if hasattr(ri, "RiElse"): + ri.RiElse.argtypes = [] + else: + ri.RiElse = _undefinedFunctionSubstitute + + if hasattr(ri, "RiElseIf"): + ri.RiElseIf.argtypes = [RtToken] + else: + ri.RiElseIf = _undefinedFunctionSubstitute + + if hasattr(ri, "RiIfBegin"): + ri.RiIfBegin.argtypes = [RtToken] + else: + ri.RiIfBegin = _undefinedFunctionSubstitute + + if hasattr(ri, "RiIfEnd"): + ri.RiIfEnd.argtypes = [] + else: + ri.RiIfEnd = _undefinedFunctionSubstitute + + if hasattr(ri, "RiMakeBrickMap"): + ri.RiMakeBrickMap.argtypes = [RtInt, POINTER(c_char_p), c_char_p] + else: + ri.RiMakeBrickMap = _undefinedFunctionSubstitute + + if hasattr(ri, "RiResource"): + ri.RiResource.argtypes = [RtToken, RtToken] + else: + ri.RiResource = _undefinedFunctionSubstitute + + if hasattr(ri, "RiResourceBegin"): + ri.RiResourceBegin.argtypes = [] + else: + ri.RiResourceBegin = _undefinedFunctionSubstitute + + if hasattr(ri, "RiResourceEnd"): + ri.RiResourceEnd.argtypes = [] + else: + ri.RiResourceEnd = _undefinedFunctionSubstitute + + if hasattr(ri, "RiScopedCoordinateSystem"): + ri.RiScopedCoordinateSystem.argtypes = [RtToken] + else: + ri.RiScopedCoordinateSystem = _undefinedFunctionSubstitute + + if hasattr(ri, "RiShader"): + ri.RiShader.argtypes = [RtToken, RtToken] + else: + ri.RiShader = _undefinedFunctionSubstitute + + if hasattr(ri, "RiSystem"): + ri.RiSystem.argtypes = [c_char_p] + else: + ri.RiSystem = _undefinedFunctionSubstitute + +def _undefinedFunctionSubstitute(*args, **kwargs): + raise NotImplementedError("This function call is not implemented by this implementation of the RenderMan interface") + def _getLastError(ri): """Getter function for the RiLastError variable. """ Modified: cgkit/trunk/cgkit/cri.py =================================================================== --- cgkit/trunk/cgkit/cri.py 2009-03-01 11:09:15 UTC (rev 296) +++ cgkit/trunk/cgkit/cri.py 2009-03-01 11:10:48 UTC (rev 297) @@ -218,6 +218,26 @@ RiLastError = property(_getRiLastError, _setRiLastError) LastError = property(_getRiLastError, _setRiLastError) + def RiArchiveBegin(self, archivename, *paramlist, **keyparams): + """Begin an inline archive. + + Example: RiArchiveBegin("myarchive") + ... + RiArchiveEnd() + RiReadArchive("myarchive") + """ + return self._ri.RiArchiveBegin(archivename, *self._createCParamList(paramlist, keyparams)) + + def RiArchiveEnd(self): + """Terminate an inline archive. + + Example: RiArchiveBegin("myarchive") + ... + RiArchiveEnd() + RiReadArchive("myarchive") + """ + self._ri.RiArchiveEnd() + def RiArchiveRecord(self, type, format, *args): """Output a user data record. @@ -327,6 +347,13 @@ bound = self._toCArray(self._ri.RtFloat, bound) self._ri.RiBound(bound) + def RiCamera(self, name, *paramlist, **keyparams): + """Mark the current camera description. + + Example: RiCamera("rightcamera") + """ + self._ri.RiCamera(name, *self._createCParamList(paramlist, keyparams)) + def RiClipping(self, near, far): """Sets the near and the far clipping plane along the direction of view. @@ -533,6 +560,13 @@ """ self._ri.RiDetailRange(minvisible, lowertransition, uppertransition, maxvisible) + def RiDisplayChannel(self, channel, *paramlist, **keyparams): + """Defines a new display channel. + + Example: RiDisplayChannel("color aovCi", "string opacity", "aovOi") + """ + self._ri.RiDisplayChannel(channel, *self._createCParamList(paramlist, keyparams)) + def RiDisk(self, height, radius, thetamax, *paramlist, **keyparams): """Create a disk (parallel to the XY plane). @@ -558,7 +592,18 @@ RiDisplay("myimage.tif", RI_FRAMEBUFFER, RI_RGB) """ self._ri.RiDisplay(name, type, mode, *self._createCParamList(paramlist, keyparams)) + + def RiElse(self): + """Add an else block to a conditional block. + """ + + self._ri.RiElse() + def RiElseIf(self, expression, *paramlist, **keyparams): + """Add an else-if block to a conditional block. + """ + self._ri.RiElseIf(expression, *self._createCParamList(paramlist, keyparams)) + def RiEnd(self): """Terminates the main block. """ @@ -672,6 +717,16 @@ """ self._ri.RiIdentity() + def RiIfBegin(self, expression, *paramlist, **keyparams): + """Begin a conditional block. + """ + self._ri.RiIfBegin(expression, *self._createCParamList(paramlist, keyparams)) + + def RiIfEnd(self): + """Terminate a conditional block. + """ + self._ri.RiIfEnd() + def RiIlluminate(self, light, onoff): """Activate or deactive a light source. @@ -704,6 +759,15 @@ """ return self._ri.RiLightSource(name, *self._createCParamList(paramlist, keyparams)) + def RiMakeBrickMap(self, ptcnames, bkmname, *paramlist, **keyparams): + """Create a brick map file from a list of point cloud file names. + + Example: RiMakeBrickMap(["sphere.ptc", "box.ptc"], "spherebox.bkm", "float maxerror", 0.002) + """ + n = len(ptcnames) + names = (n*ctypes.c_char_p)(*ptcnames) + self._ri.RiMakeBrickMap(n, names, bkmname, *self._createCParamList(paramlist, keyparams)) + def RiMakeCubeFaceEnvironment(self, px,nx,py,ny,pz,nz, texname, fov, filterfunc, swidth, twidth, *paramlist, **keyparams): """Convert six image files into an environment map. @@ -1116,6 +1180,21 @@ Example: RiRelativeDetail(0.7)""" self._ri.RiRelativeDetail(relativedetail) + def RiResource(self, handle, type, *paramlist, **keyparams): + """Create or operate on a named resource of a particular type. + """ + self._ri.RiResource(handle, type, *self._createCParamList(paramlist, keyparams)) + + def RiResourceBegin(self): + """Push the current set of resources. + """ + self._ri.RiResourceBegin() + + def RiResourceEnd(self): + """Pop the current set of resources. + """ + self._ri.RiResourceEnd() + def RiReverseOrientation(self): """Causes the current orientation to be toggled. @@ -1142,6 +1221,13 @@ scale = self._toCArray(self._ri.RtFloat, scale) self._ri.RiScale(*tuple(scale)) + def RiScopedCoordinateSystem(self, spacename): + """Mark the current coordinate system with a name but store it on a separate stack. + + Example: RiScopedCoordinateSystem("lamptop") + """ + self._ri.RiScopedCoordinateSystem(spacename) + def RiScreenWindow(self, left, right, bottom, top): """Specify the extents of the output image on the image plane. @@ -1149,6 +1235,13 @@ """ self._ri.RiScreenWindow(left, right, bottom, top) + def RiShader(self, name, handle, *paramlist, **keyparams): + """Set the current coshader. + + Example: RiShader("plastic", "plastic_layer", Kd=0.7, Ks=0.3) + """ + self._ri.RiShader(name, handle, *self._createCParamList(paramlist, keyparams)) + def RiShadingInterpolation(self, type): """Specify how shading samples are interpolated. @@ -1255,6 +1348,11 @@ Example: RiSurface("plastic", Kd=0.7, Ks=0.3)""" self._ri.RiSurface(name, *self._createCParamList(paramlist, keyparams)) + def RiSystem(self, cmd): + """Execute an arbitrary command in the same environment as the current rendering pass. + """ + self._ri.RiSystem(cmd) + def RiTextureCoordinates(self, s1, t1, s2, t2, s3, t3, s4, t4): """Set the current set of texture coordinates. Modified: cgkit/trunk/cgkit/ri.py =================================================================== --- cgkit/trunk/cgkit/ri.py 2009-03-01 11:09:15 UTC (rev 296) +++ cgkit/trunk/cgkit/ri.py 2009-03-01 11:10:48 UTC (rev 297) @@ -280,7 +280,8 @@ The version number is automatically placed into the stream before any "real" Ri calls are made. Output from RiArchiveRecord() will - be placed before the version number. + be placed before the version number. (Note: The version line is disabled + for now). """ def __init__(self, outstream): @@ -299,7 +300,9 @@ def write(self, data): """Write data into the stream.""" if self.output_version: - self.out.write('version 3.03\n') + # The binding contains newer calls, so this version number + # might not be accurate anyway. +# self.out.write('version 3.03\n') self.output_version = 0 self.out.write(data) @@ -1029,6 +1032,15 @@ _ribout.write('ShadingInterpolation "'+type+'"\n') +# RiShader +def RiShader(name, handle, *paramlist, **keyparams): + """Set the current coshader. + + Example: RiShader("plastic", "plastic_layer", Kd=0.7, Ks=0.3)""" + + _ribout.write('Shader "'+name+'"'+' "'+handle+'"'+ \ + _paramlist2string(paramlist, keyparams)+"\n") + # RiSurface def RiSurface(name, *paramlist, **keyparams): """Set the current surface shader. @@ -1128,6 +1140,14 @@ _ribout.write('Display "'+name+'" "'+type+'" "'+mode+'"'+ \ _paramlist2string(paramlist, keyparams)+"\n") +# RiDisplayChannel +def RiDisplayChannel(channel, *paramlist, **keyparams): + """Defines a new display channel. + + Example: RiDisplayChannel("color aovCi", "string opacity", "aovOi") + """ + _ribout.write('DisplayChannel "%s"%s\n'%(channel, _paramlist2string(paramlist, keyparams))) + # RiFormat def RiFormat(xres, yres, aspect): """Set the resolution of the output image and the aspect ratio of a pixel. @@ -1171,6 +1191,14 @@ _ribout.write('Projection "'+name+'"'+ \ _paramlist2string(paramlist, keyparams)+"\n") +# RiCamera +def RiCamera(name, *paramlist, **keyparams): + """Mark the current camera description. + + Example: RiCamera("rightcamera") + """ + _ribout.write('Camera "%s"%s\n'%(name, _paramlist2string(paramlist, keyparams))) + # RiScreenWindow def RiScreenWindow(left, right, bottom, top): """Specify the extents of the output image on the image plane. @@ -1624,11 +1652,34 @@ In this implementation the callback function is not used and can be left out. - RiExample: RiReadArchive("teapot.rib")""" + Example: RiReadArchive("teapot.rib")""" _ribout.write('ReadArchive "'+filename+'"\n') +# RiArchiveBegin +def RiArchiveBegin(archivename, *paramlist, **keyparams): + """Begin an inline archive. + + Example: RiArchiveBegin("myarchive") + ... + RiArchiveEnd() + RiReadArchive("myarchive") + """ + _ribout.write('ArchiveBegin "%s"%s\n'%(archivename, _paramlist2string((), keyparams))) + return archivename +# RiArchiveEnd +def RiArchiveEnd(): + """Terminate an inline archive. + + Example: RiArchiveBegin("myarchive") + ... + RiArchiveEnd() + RiReadArchive("myarchive") + """ + _ribout.write('ArchiveEnd\n') + + def RiProcDelayedReadArchive(): return "DelayedReadArchive" def RiProcRunProgram(): return "RunProgram" def RiProcDynamicLoad(): return "DynamicLoad" @@ -1858,6 +1909,15 @@ _ribout.write('MakeShadow "'+picname+'" "'+shadowname+'"'+_paramlist2string(paramlist, keyparams)+'\n') +# RiMakeBrickMap +def RiMakeBrickMap(ptcnames, bkmname, *paramlist, **keyparams): + """Create a brick map file from a list of point cloud file names. + + Example: RiMakeBrickMap(["sphere.ptc", "box.ptc"], "spherebox.bkm", "float maxerror", 0.002) + """ + names = " ".join(map(lambda name: '"%s"'%name, ptcnames)) + _ribout.write('MakeBrickMap [%s] "%s"%s\n'%(names, bkmname, _paramlist2string(paramlist, keyparams))) + # RiDetail def RiDetail(bound): """Set the current bounding box. @@ -1905,9 +1965,16 @@ Example: RiCoordinateSystem("lamptop") """ - _ribout.write('CoordinateSystem "'+spacename+'"\n') +# RiScopedCoordinateSystem +def RiScopedCoordinateSystem(spacename): + """Mark the current coordinate system with a name but store it on a separate stack. + + Example: RiScopedCoordinateSystem("lamptop") + """ + _ribout.write('ScopedCoordinateSystem "'+spacename+'"\n') + # RiTransformPoints def RiTransformPoints(fromspace, tospace, points): """Transform a set of points from one space to another. @@ -1949,6 +2016,59 @@ return _current_context +# RiSystem +def RiSystem(cmd): + """Execute an arbitrary command in the same environment as the current rendering pass. + """ + # Escape quotes + cmd = cmd.replace('"', r'\"') + _ribout.write('System "%s"\n'%cmd) + +# RiIfBegin +def RiIfBegin(expression, *paramlist, **keyparams): + """Begin a conditional block. + """ + _ribout.write('IfBegin "%s"%s\n'%(expression, _paramlist2string(paramlist, keyparams))) + +# RiElseIf +def RiElseIf(expression, *paramlist, **keyparams): + """Add an else-if block to a conditional block. + """ + + _ribout.write('ElseIf "%s"%s\n'%(expression, _paramlist2string(paramlist, keyparams))) + +# RiElse +def RiElse(): + """Add an else block to a conditional block. + """ + + _ribout.write('Else\n') + +# RiIfEnd +def RiIfEnd(): + """Terminate a conditional block. + """ + + _ribout.write('IfEnd\n') + +# RiResource +def RiResource(handle, type, *paramlist, **keyparams): + """Create or operate on a named resource of a particular type. + """ + _ribout.write('Resource "%s" "%s"%s\n'%(handle, type, _paramlist2string(paramlist, keyparams))) + +# RiResourceBegin +def RiResourceBegin(): + """Push the current set of resources. + """ + _ribout.write('ResourceBegin\n') + +# RiResourceEnd +def RiResourceEnd(): + """Pop the current set of resources. + """ + _ribout.write('ResourceEnd\n') + ##################### Global variabels (internal) #################### _contexts = {} Modified: cgkit/trunk/doc/tex/ri.tex =================================================================== --- cgkit/trunk/doc/tex/ri.tex 2009-03-01 11:09:15 UTC (rev 296) +++ cgkit/trunk/doc/tex/ri.tex 2009-03-01 11:10:48 UTC (rev 297) @@ -25,9 +25,8 @@ This document is not an introduction to the RenderMan interface itself, it just explains the usage of this particular Python binding. The binding was written to be compliant to v3.2 of Pixar's -RenderMan Interface specification. However, it also supports some -newer features such as string handles for light sources or object -instances. +RenderMan Interface specification. However, it also supports features +that were introduced after v3.2. There is another RenderMan module called \refmodule[cgkit.cri]{cri} that interfaces a renderer directly. Almost everything that is said in this section @@ -334,7 +333,8 @@ If this option is set to 0 directly after \function{RiBegin()} is called, then no \code{"version"} call will be generated in the RIB stream (default is 1).\\ -New in version 1.1. +New in version 1.1.\\ +(as of cgkit 2.0.0alpha9, the \code{version} call has been disabled) \end{funcdesc} %----------- This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. 
 [cgkit-commits] SF.net SVN: cgkit:[312] cgkit/trunk From: - 2009-05-09 10:00:36 Revision: 312 http://cgkit.svn.sourceforge.net/cgkit/?rev=312&view=rev Author: mbaas Date: 2009-05-09 10:00:26 +0000 (Sat, 09 May 2009) Log Message: ----------- RMShader: Passing array parameters via the constructor didn't work. Modified Paths: -------------- cgkit/trunk/cgkit/rmshader.py cgkit/trunk/changelog.txt Added Paths: ----------- cgkit/trunk/unittests/test_rmshader.py Modified: cgkit/trunk/cgkit/rmshader.py =================================================================== --- cgkit/trunk/cgkit/rmshader.py 2009-05-09 09:58:50 UTC (rev 311) +++ cgkit/trunk/cgkit/rmshader.py 2009-05-09 10:00:26 UTC (rev 312) @@ -280,32 +280,9 @@ # print 'declare("%s", type=%s, cls=%s, arraysize=%s, default=%s)'%(name, type, cls, arraysize, default) - # Create a "dummy shader" which will be passed to slparams to parse - # the declaration in name - shd = "surface spam(%s) {}"%name - try: - # Force a syntax error when name contains no declaration - if " " not in name: - raise slparams.SyntaxError() - slinfo = slparams.slparams(StringIO.StringIO(shd)) - shdtype, shdname, params = slinfo[0] - except slparams.SyntaxError, e: - # Check if name is only a single name or if there was an attempt - # to specify the entire declaration - invalid = " []():;'\"'" - for inv in invalid: - if inv in name: - raise ValueError, 'Invalid declaration: "%s"'%name - # It's probably really just the name, so use the remaining - # arguments to create a parameter tuple... - if cls==None: - cls = "uniform" - if type==None: - raise ValueError, 'No type for parameter "%s" specified'%name - if type not in ["float", "string", "color", "point", "vector", - "normal", "matrix"]: - raise ValueError, 'Invalid type for parameter "%s": %s'%(name, type) - params = [("", cls, type, arraysize, name, "", str(default))] + # Get a slparams-stype params tuple from either name alone or from + # all the args + params = self._declare_getParams(name, type, cls, arraysize, default) typelut = {"float":"double", "string":"str", @@ -323,7 +300,7 @@ pname = p[4] pdefault = slparams.convertdefault(p) slottype = typelut[ptype] - if parraylen==None: + if parraylen is None: decl = "%s %s"%(p[1], ptype) else: decl = "%s %s[%d]"%(p[1], ptype, parraylen) @@ -344,7 +321,18 @@ if pytype=="str": pdefault = self.undeclared[pname] else: - pdefault = eval("%s(%s)"%(pytype, repr(self.undeclared[pname]))) + # Scalar? + if parraylen is None: + pdefault = eval("%s(%s)"%(pytype, repr(self.undeclared[pname]))) + # Array + else: + userDefault = self.undeclared[pname] + try: + if len(userDefault)!=parraylen: + raise ValueError('Invalid default value for shader parameter "%s". Expected %s values, but got %s'%(pname, parraylen, len(userDefault))) + except TypeError: + raise TypeError('Invalid default value for shader parameter "%s". Expected a sequence of %s values.'%(pname, parraylen)) + pdefault = [eval("%s(%s)"%(pytype, repr(v))) for v in self.undeclared[pname]] del self.undeclared[pname] # Create the slot and add the variable to the params dictionary @@ -353,7 +341,45 @@ else: self.createSlot(pname, slottype, parraylen, pdefault) self.shaderparams[pname] = decl + + def _declare_getParams(self, name, type, cls, arraysize, default): + """Helper method for declare(). + Turns the arguments into a params "tuple". See declare() for a description + of the arguments. + The return value is either a params object as returned by slparams() + or an old-style params tuple. + """ + # Create a "dummy shader" which will be passed to slparams to parse + # the declaration in name + shd = "surface spam(%s) {}"%name + try: + # Force a syntax error when name contains no declaration + if " " not in name: + raise slparams.SyntaxError() + slinfo = slparams.slparams(StringIO.StringIO(shd)) + shdtype, shdname, params = slinfo[0] + except slparams.SyntaxError, e: + # Check if name is only a single name or if there was an attempt + # to specify the entire declaration + invalid = " []():;'\"'" + for inv in invalid: + if inv in name: + raise ValueError, 'Invalid declaration: "%s"'%name + # It's probably really just the name, so use the remaining + # arguments to create a parameter tuple... + if cls is None: + cls = "uniform" + if type is None: + raise ValueError, 'No type for parameter "%s" specified'%name + if type not in ["float", "string", "color", "point", "vector", + "normal", "matrix"]: + raise ValueError, 'Invalid type for parameter "%s": %s'%(name, type) + params = [("", cls, type, arraysize, name, "", str(default))] + + return params + + # loadShaderSource def loadShaderSource(self): """Load shader source and replace the shader name. @@ -365,14 +391,14 @@ \return Shader source code or None """ - if self.filename==None: + if self.filename is None: return None f = file(self.filename) src = f.read() # Search for + one or more white space + .. match = re.search("%s\s+%s"%(self.shadertype, self.shadername), src) - if match!=None: + if match is not None: s, e = match.start(), match.end() src = src[:e-len(self.shadername)] + "$SHADERNAME" + src[e:] else: @@ -393,14 +419,14 @@ \param default Default value or None """ - if arraylen==None: + if arraylen is None: exec "slot = %sSlot()"%type.capitalize() - if default!=None: + if default is not None: slot.setValue(default) else: exec "slot = %sArraySlot()"%type.capitalize() slot.resize(arraylen) - if default!=None: + if default is not None: for i,v in enumerate(default): slot[i] = v Modified: cgkit/trunk/changelog.txt =================================================================== --- cgkit/trunk/changelog.txt 2009-05-09 09:58:50 UTC (rev 311) +++ cgkit/trunk/changelog.txt 2009-05-09 10:00:26 UTC (rev 312) @@ -10,6 +10,8 @@ Bug fixes/enhancements: +- RMShader: Passing parameter values for array parameters in the constructor + didn't work. - ri/cri: Added the following calls: RiArchiveBegin(), RiArchiveEnd(), RiCamera(), RiDisplayChannel(), RiIfBegin(), RiElseIf(), RiElse(), RiIfEnd(), Added: cgkit/trunk/unittests/test_rmshader.py =================================================================== --- cgkit/trunk/unittests/test_rmshader.py (rev 0) +++ cgkit/trunk/unittests/test_rmshader.py 2009-05-09 10:00:26 UTC (rev 312) @@ -0,0 +1,61 @@ +# Test the RMShader class + +import unittest +from cgkit.rmshader import RMShader +from cgkit.cgtypes import * +import numpy + +class TestRMShader(unittest.TestCase): + """Test the RMShader class. + """ + + def testShader(self): + """Test the RMShader class. + """ + shd = RMShader("data/testshader.sl") + # Attribute access + self.assertEqual(1.0, shd.Ka) + self.assertEqual((0,0,0), shd.norm) + self.assertEqual([1.0, 2.0], shd.uv) + self.assertEqual((0,0,0), shd.out) + self.assertEqual([(1.0,2.0,3.0), (4.0,5.0,6.0), (7.0,8.0,9.0)], shd.pnts) + self.assertEqual((1.0,1.0,1.0), shd.col) + self.assertEqual(mat4(1), shd.M) + self.assertEqual(0.5, shd.Kd) + self.assertEqual([0.5], shd.singleArr) + + # Slot access + self.assertEqual(1.0, shd.Ka_slot.getValue()) + self.assertEqual((0,0,0), shd.norm_slot.getValue()) + self.assertEqual(1.0, shd.uv_slot.getValue(0)) + self.assertEqual(2.0, shd.uv_slot.getValue(1)) + self.assertEqual((0,0,0), shd.out_slot.getValue()) + self.assertEqual((1.0,2.0,3.0), shd.pnts_slot.getValue(0)) + self.assertEqual((4.0,5.0,6.0), shd.pnts_slot.getValue(1)) + self.assertEqual((7.0,8.0,9.0), shd.pnts_slot.getValue(2)) + self.assertEqual((1.0,1.0,1.0), shd.col_slot.getValue()) + self.assertEqual(mat4(1), shd.M_slot.getValue()) + self.assertEqual(0.5, shd.Kd_slot.getValue()) + self.assertEqual(0.5, shd.singleArr_slot.getValue(0)) + + # Assign some new values + shd.Ka = 0.7 + self.assertEqual(0.7, shd.Ka_slot.getValue()) + shd.singleArr = [0.6] + self.assertEqual(0.6, shd.singleArr_slot.getValue(0)) + + def testConstructor(self): + """Test passing initial parameter values in the constructor. + """ + shd = RMShader("data/testshader.sl", Ka=0.7, norm=(0,0,1), uv=(0.2,0.3), singleArr=[0.8]) + self.assertEqual(0.7, shd.Ka_slot.getValue()) + self.assertEqual((0,0,1), shd.norm_slot.getValue()) + self.assertEqual(0.2, shd.uv_slot.getValue(0)) + self.assertEqual(0.3, shd.uv_slot.getValue(1)) + self.assertEqual(0.8, shd.singleArr_slot.getValue(0)) + + +###################################################################### + +if __name__=="__main__": + unittest.main() This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.   [cgkit-commits] SF.net SVN: cgkit:[317] cgkit/trunk From: - 2009-05-29 18:51:28 Revision: 317 http://cgkit.svn.sourceforge.net/cgkit/?rev=317&view=rev Author: mbaas Date: 2009-05-29 18:51:16 +0000 (Fri, 29 May 2009) Log Message: ----------- Wrapped the mat4 constructor that takes 16 floats as input (copy.copy() now works on core mat4 objects). Modified Paths: -------------- cgkit/trunk/changelog.txt cgkit/trunk/setup.py cgkit/trunk/unittests/test_mat4.py cgkit/trunk/wrappers/py_mat4.cpp Modified: cgkit/trunk/changelog.txt =================================================================== --- cgkit/trunk/changelog.txt 2009-05-27 13:28:39 UTC (rev 316) +++ cgkit/trunk/changelog.txt 2009-05-29 18:51:16 UTC (rev 317) @@ -10,6 +10,9 @@ Bug fixes/enhancements: +- mat4: Wrapped the mat4 constructor that takes 16 floats as input. Because + this one was missing, calling copy.copy() on a mat4 that was returned by + the library (instead of being created by the user) did not work. - RMShader: Passing parameter values for array parameters in the constructor didn't work. - ri/cri: Added the following calls: Modified: cgkit/trunk/setup.py =================================================================== --- cgkit/trunk/setup.py 2009-05-27 13:28:39 UTC (rev 316) +++ cgkit/trunk/setup.py 2009-05-29 18:51:16 UTC (rev 317) @@ -204,7 +204,7 @@ LIBS = [] CC_ARGS = [] LINK_ARGS = [] -MACROS = [] +MACROS = [("BOOST_PYTHON_MAX_ARITY", 18)] data_files = [] scripts = [] Modified: cgkit/trunk/unittests/test_mat4.py =================================================================== --- cgkit/trunk/unittests/test_mat4.py 2009-05-27 13:28:39 UTC (rev 316) +++ cgkit/trunk/unittests/test_mat4.py 2009-05-29 18:51:16 UTC (rev 317) @@ -3,7 +3,7 @@ import unittest from cgkit.cgtypes import * #from cgkit.light.cgtypes import * -import math, os, pickle, cPickle, sys +import math, os, pickle, cPickle, sys, copy class TestMat4(unittest.TestCase): @@ -616,7 +616,15 @@ self.assertEqual(m, n) + ###################################################################### + def testCopy(self): + """Test copying an internal mat4. + """ + m = mat4.translation((1,2,3)) + m2 = copy.copy(m) + self.assertEqual(m, m2) + ###################################################################### setEpsilon(1E-12) Modified: cgkit/trunk/wrappers/py_mat4.cpp =================================================================== --- cgkit/trunk/wrappers/py_mat4.cpp 2009-05-27 13:28:39 UTC (rev 316) +++ cgkit/trunk/wrappers/py_mat4.cpp 2009-05-29 18:51:16 UTC (rev 317) @@ -208,10 +208,10 @@ class_("mat4") .def(init()) - /* .def(init())*/ + double,double,double,double, + double,double,double,double>()) .def(init()) .def(self_ns::str(self)) .def("__repr__", &repr) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.   [cgkit-commits] SF.net SVN: cgkit:[322] cgkit/trunk From: - 2009-06-24 09:21:10 Revision: 322 http://cgkit.svn.sourceforge.net/cgkit/?rev=322&view=rev Author: mbaas Date: 2009-06-24 09:20:36 +0000 (Wed, 24 Jun 2009) Log Message: ----------- Read a version-specific config file in addition to the general config file. Modified Paths: -------------- cgkit/trunk/changelog.txt cgkit/trunk/setup.py Modified: cgkit/trunk/changelog.txt =================================================================== --- cgkit/trunk/changelog.txt 2009-06-21 07:42:06 UTC (rev 321) +++ cgkit/trunk/changelog.txt 2009-06-24 09:20:36 UTC (rev 322) @@ -10,6 +10,8 @@ Bug fixes/enhancements: +- setup script: In addition to the general config file config.cfg the setup + script also reads a version-specific config_pyXY.cfg script. - mat4: Wrapped the mat4 constructor that takes 16 floats as input. Because this one was missing, calling copy.copy() on a mat4 that was returned by the library (instead of being created by the user) did not work. Modified: cgkit/trunk/setup.py =================================================================== --- cgkit/trunk/setup.py 2009-06-21 07:42:06 UTC (rev 321) +++ cgkit/trunk/setup.py 2009-06-24 09:20:36 UTC (rev 322) @@ -241,14 +241,23 @@ vars = get_config_vars() vars["OPT"] = vars["OPT"].replace("-g", "") -# Read the config file -if os.path.exists("config.cfg"): - exec(open("config.cfg").read()) +# Read the general config file +configName = "config.cfg" +if os.path.exists(configName): + print ("Reading %s..."%configName) + exec(open(configName).read()) else: - print (70*"-") - print ("Warning: No config file available (config.cfg)") - print (70*"-") + print ("Config file (%s) not available"%configName) +# Read the version-specific config file +vi = sys.version_info +configName = "config_py%s%s.cfg"%vi[:2] +if os.path.exists(configName): + print ("Reading %s..."%configName) + exec(open(configName).read()) +else: + print ("Version-specific config file (%s) not available"%configName) + # Disable all optional libs, STL and Boost for the light version... if INSTALL_CGKIT_LIGHT: LIB3DS_AVAILABLE = False This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.   [cgkit-commits] SF.net SVN: cgkit:[329] cgkit/trunk From: - 2009-08-06 20:01:41 Revision: 329 http://cgkit.svn.sourceforge.net/cgkit/?rev=329&view=rev Author: mbaas Date: 2009-08-06 20:01:27 +0000 (Thu, 06 Aug 2009) Log Message: ----------- Support for custom RIB strings on light sources (so far, image pass only). Modified Paths: -------------- cgkit/trunk/cgkit/ribexport.py cgkit/trunk/changelog.txt Modified: cgkit/trunk/cgkit/ribexport.py =================================================================== --- cgkit/trunk/cgkit/ribexport.py 2009-07-11 13:14:56 UTC (rev 328) +++ cgkit/trunk/cgkit/ribexport.py 2009-08-06 20:01:27 UTC (rev 329) @@ -781,6 +781,10 @@ RiAttributeBegin() RiAttribute("identifier", "name", lgt.name) RiConcatTransform(lgt.worldtransform.toList()) + # Custom RIB + rib = getattr(lgt, "rib", None) + if rib is not None: + RiArchiveRecord(RI_VERBATIM, rib+"\n") lid = self.exporter.applyLightSource(lgt) RiAttributeEnd() if lid!=None: Modified: cgkit/trunk/changelog.txt =================================================================== --- cgkit/trunk/changelog.txt 2009-07-11 13:14:56 UTC (rev 328) +++ cgkit/trunk/changelog.txt 2009-08-06 20:01:27 UTC (rev 329) @@ -7,9 +7,15 @@ - New module sloargs to parse compiled RenderMan shaders. - New module pointcloud to read/write RenderMan point cloud files. +- New module sequence that contains functionality to deal with numbered + (file) sequences. There are also new utilities seqls.py, seqmv.py, seqcp.py + and seqrm.py (the equivalent of ls, mv, cp, rm but they work on file + sequences). Bug fixes/enhancements: +- ribexport: A custom rib string can now also be added to light sources + (for the image pass). - setup script: In addition to the general config file config.cfg the setup script also reads a version-specific config_pyXY.cfg script. - mat4: Wrapped the mat4 constructor that takes 16 floats as input. Because This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.   [cgkit-commits] SF.net SVN: cgkit:[330] cgkit/trunk From: - 2009-08-08 09:09:46 Revision: 330 http://cgkit.svn.sourceforge.net/cgkit/?rev=330&view=rev Author: mbaas Date: 2009-08-08 09:09:38 +0000 (Sat, 08 Aug 2009) Log Message: ----------- Brushed up the sequence module (and utils), wrote docs, unit tests and fixed all issues found by the tests. Modified Paths: -------------- cgkit/trunk/cgkit/sequence.py cgkit/trunk/utilities/seqcp.py cgkit/trunk/utilities/seqls.py cgkit/trunk/utilities/seqmv.py cgkit/trunk/utilities/seqrm.py Added Paths: ----------- cgkit/trunk/unittests/test_sequence.py Modified: cgkit/trunk/cgkit/sequence.py =================================================================== --- cgkit/trunk/cgkit/sequence.py 2009-08-06 20:01:27 UTC (rev 329) +++ cgkit/trunk/cgkit/sequence.py 2009-08-08 09:09:38 UTC (rev 330) @@ -45,13 +45,13 @@ class SeqString: """Sequence string class. - Sequence strings treat numbers inside a strings as integer numbers + Sequence strings treat numbers inside strings as integer numbers and not as strings. This can be used to sort numerically (e.g. - 'anim01' is smaller than 'anim0002'). + anim01 is smaller than anim0002). A sequence string is initialized by passing a regular string to - the constructor or by calling setString(). - The main task of a SeqString is comparing two strings which can + the constructor. It can be converted back using the :func:str() operator. + The main task of a :class:SeqString is comparing two strings which can be done with the normal comparison operators. Example: >>> a = SeqString('a08') @@ -71,17 +71,17 @@ to an empty string. """ # This is an alternating sequence of text and number values - # (always beginning with a text (which might be empty)). + # (always beginning and ending with a text (which might both be empty)). # The value part is a tuple (value,numdigits) where value # is an integer and numdigits the number of digits the - # value was made of. + # value was made of. The list can never be empty (it always contains + # at least one string, even when that one is empty). # Example: 'anim1_0001.png' -> ['anim', (1,1), '_', (1,4), '.png'] - self._value = [] - + self._value = [""] self._initSeqString(s) def __repr__(self): - return self.__str__() + return "'%s'"%self.__str__() def __str__(self): """Convert the sequence string into a normal string. @@ -98,7 +98,7 @@ val,ndigits = vn a = '%'+"0%dd"%ndigits res += a%val - return res + return res def __cmp__(self, other): """Comparison operator. @@ -106,19 +106,27 @@ The text parts are treated as strings, the number parts as numbers (e.g. 'a08' is greater than 'a2'). """ - if other is None: return 1 + if not isinstance(other, SeqString): + if not isinstance(other, basestring): + return 1 + # Convert both strings into pristine SeqStrings (because some numbers + # on the input strings may have been replaced by strings which would + # mess with the comparison). + selfStr = SeqString(self) + other = SeqString(other) + # Check the 'structure' of the strings first. # The numeric comparison is only done when the strings have the same # text/num patterns. - res = self.match_cmp(other) + res = selfStr.match_cmp(other) if res!=0: return res # Compare the individual components of the values side by side - for i, (a,b) in enumerate(zip(self._value, other._value)): + for i, (a,b) in enumerate(zip(selfStr._value, other._value)): if i%2==1: # Get the numbers a = a[0] @@ -131,7 +139,7 @@ # If we are here everything has been equal so far, but maybe # one string has one component more in _value - return cmp(len(self._value), len(other._value)) + return cmp(len(selfStr._value), len(other._value)) def _initSeqString(self, s): """Initialize the sequence string with a string. @@ -183,20 +191,24 @@ else: numtup = (int(textbuf),numtup[1]) res.append(numtup) + res.append("") self._value = res def match(self, template, numPos=None): """Check if one sequence string is equal to another except for one or all numbers. - Returns True if the text parts of self and template are equal, - i.e. both strings belong to the same sequence. + Returns True if the text parts of *self* and *template* are equal, + i.e. both strings belong to the same sequence. *template* must be + a :class:SeqString object. - numPos is the index of the number that is allowed to vary. For example, - if numPos is -1, the last number in a string may be different for two - strings to be in the same sequence. If numPos is None, all numbers - may vary. + *numPos* is the index of the number that is allowed to vary. For example, + if *numPos* is -1, only the last number in a string may be different for two + strings to be in the same sequence. Al other numbers must match exactly + (including the padding). If *numPos* is None, all numbers may vary. """ + if not isinstance(template, SeqString): + raise TypeError("The template argument must be a SeqString object") # The lengths of the value lists must be equal if len(self._value)!=len(template._value): @@ -212,7 +224,7 @@ if i%2==0: if va!=vb: return False - elif numPos is not None and i!=numPos and va[0]!=vb[0]: + elif numPos is not None and i!=numPos and va!=vb: return False return True @@ -220,12 +232,13 @@ def match_cmp(self, template): """Comparison function to build groups. - This is the same as match() but with a different return value - so that this method can be used as comparison function for sort(). + Compare the text parts (the group name) of two sequence strings. + Numbers within the strings are ignored. - 0: self==template, <0: self0: self>template + 0 is returned if *self* and *template* belong to the same group, + a negative value is returned if *self* comes before *template* and + a positive value is returned if *self* comes after *template*. """ - a = self.groupRepr() b = template.groupRepr() return cmp(a,b) @@ -233,8 +246,8 @@ def groupRepr(self, numChar="*"): """Return a template string where the numbers are replaced by the given character. """ - res="" + numChar = str(numChar) for i,v in enumerate(self._value): if i%2==0: res += v @@ -245,19 +258,21 @@ def numCount(self): """Return the number of number occurrences in the string. - Example: 'anim01.tif' -> 1 - 'anim1_018.tif' -> 2 - 'anim' -> 0 + Examples: + + - anim01.tif -> 1 + - anim1_018.tif -> 2 + - anim -> 0 """ return int(len(self._value)/2) def getNum(self, idx): """Return a particular number inside the string. - idx is the index of the number (0-based) which may also be + *idx* is the index of the number (0-based) which may also be negative. The return value is an integer containing the number at that position. - Raises an IndexError exception when idx is out of range. + Raises an :exc:IndexError exception when *idx* is out of range. """ if idx<0: @@ -270,10 +285,10 @@ def getNumStr(self, idx): """Return a particular number as a string just as it appears in the original string. - idx is the index of the number (0-based) which may also be + *idx* is the index of the number (0-based) which may also be negative. The return value is a string that contains the number as it appears in the string (including padding). - Raises an IndexError exception when idx is out of range. + Raises an :exc:IndexError exception when *idx* is out of range. """ if idx<0: @@ -296,29 +311,35 @@ return res - def setNum(self, idx, value): + def setNum(self, idx, value, width=None): """Set a new number. - idx is the index of the number (may be negative) and value - is the new integer value. - Raises an IndexError exception when idx is out of range. + *idx* is the index of the number (may be negative) and *value* + is the new integer value. If *width* is given, it will be the new + width of the number, otherwise the number keeps its old width. + Raises an :exc:IndexError exception when *idx* is out of range. + + Note: It is possible to set a negative value. But when converted to + a string and then back to a sequence string again, that negative + number becomes a positive number and the minus symbol is part of + the preceding text part. """ - if idx<0: idx = self.numCount()+idx if idx<0 or idx>=self.numCount(): raise IndexError, "index out of range" - value = int(value) - width = self._value[idx*2+1][1] - self._value[idx*2+1] = (value,width) + if width is None: + width = self._value[idx*2+1][1] + self._value[idx*2+1] = (int(value),int(width)) def setNums(self, nums): """Set all numbers at once. - nums is a list of integers. The number of values may not - exceed the number count in the string, otherwise an IndexError - exception is thrown. + *nums* is a list of integers. The number of values may not + exceed the number count in the string, otherwise an :exc:IndexError + exception is thrown. There may be fewer items in *nums* though in + which case the remaining numbers in the string keep their old value. """ for i,val in enumerate(nums): self.setNum(i, val) @@ -326,10 +347,9 @@ def getNumWidth(self, idx): """Return the number of digits of a particular number. - idx is the index of the number (may be negative). - Raises an IndexError exception when idx is out of range. + *idx* is the index of the number (may be negative). + Raises an :meth:IndexError exception when *idx* is out of range. """ - if idx<0: idx = self.numCount()+idx if idx<0 or idx>=self.numCount(): @@ -340,11 +360,10 @@ def setNumWidth(self, idx, width): """Set the number of digits of a number. - idx is the index of the number (may be negative) and width + *idx* is the index of the number (may be negative) and *width* the new number of digits. - Raises an IndexError exception when idx is out of range. + Raises an :exc:IndexError exception when *idx* is out of range. """ - if idx<0: idx = self.numCount()+idx if idx<0 or idx>=self.numCount(): @@ -368,8 +387,8 @@ def setNumWidths(self, widths): """Set the number of digits for all numbers. - widths must be a list of integers. The number of values may not - exceed the number count in the string, otherwise an IndexError + *widths* must be a list of integers. The number of values may not + exceed the number count in the string, otherwise an :exc:IndexError exception is thrown. """ for i,w in enumerate(widths): @@ -378,20 +397,23 @@ def deleteNum(self, idx): """Delete a number inside the string. - idx is the index of the number (0-based) which may also be + This is the same as replacing the number by an empty string. + + *idx* is the index of the number (0-based) which may also be negative. - Raises an IndexError exception when idx is out of range. + Raises an :exc:IndexError exception when *idx* is out of range. """ self.replaceNum(idx, "") def replaceNum(self, idx, txt): """Replace a number by a string. - idx is the index of the number (0-based) which may also be - negative. txt is a string that will replace the number. - Raises an IndexError exception when idx is out of range. + The string is merged with the surrounding string parts. + + *idx* is the index of the number (0-based) which may also be + negative. *txt* is a string that will replace the number. + Raises an :exc:IndexError exception when *idx* is out of range. """ - if idx<0: idx = self.numCount()+idx if idx<0 or idx>=self.numCount(): @@ -408,9 +430,9 @@ def replaceStr(self, idx, txt): """Replace a string part by another string. - idx is the index of the sub-string (0-based) which may also be - negative. txt is a string that will replace the sub-string. - Raises an IndexError exception when idx is out of range. + *idx* is the index of the sub-string (0-based) which may also be + negative. *txt* is a string that will replace the sub-string. + Raises an :exc:IndexError exception when *idx* is out of range. """ if idx<0: @@ -426,18 +448,19 @@ raise IndexError, "index out of range" # Replace the text - self._value[idx2] = txt + self._value[idx2] = str(txt) class Sequence: """A list of names/objects that all belong to the same sequence. The sequence can store the original objects that are associated with a - name or it can only store the names (as SeqString) objects. If the - original objects are available or not depends on how the sequence was - built. If the getNameFunc parameter was used when building the sequence - (see Sequences), then the original objects will be available. + name or it can only store the names (as :class:SeqString objects). + Whether the original objects are available or not depends on how the + sequence was built. If the *nameFunc* parameter was used when building + the sequence (see :func:buildSequences), then the original objects will be available. - The class can be used like a list (using len(), index operator or iteration). + The class can be used like a list (using :func:len(), index operator or + iteration). """ def __init__(self): @@ -455,7 +478,13 @@ if len(ranges)==0: return placeholder else: - return "%s (%s)"%(placeholder, "; ".join(ranges)) + infoStr = "; ".join(ranges) + if len(infoStr)>20: + infoStr = "%d items"%len(self._names) + return "%s (%s)"%(placeholder, infoStr) + + def __repr__(self): + return ""%self.__str__() def __len__(self): """Return the length of the sequence. @@ -475,17 +504,17 @@ return self._objects[idx] def iterNames(self): - """Iterate over the names. + """Iterates over the object names. - Yields SeqString objects. + Yields :class:SeqString objects. """ return iter(self._names) def iterObjects(self): - """Yield over the objects. + """Iterate over the objects. - Yields the original objects or the names as SeqStrings if the - objects haven't been stored in the sequence. + Yields the original objects or the names as :class:SeqString objects + if the objects haven't been stored in the sequence. Using this method is equivalent to iterating over the sequence object directly. """ @@ -497,26 +526,48 @@ def match(self, name, numPos=None): """Check if a name matches the names in this sequence. + *name* is a string or :class:SeqString object that is tested if + it matches the names in the sequence. If the sequence doesn't contain any name at all yet, then any name will match. + + *numPos* is an integer that specifies which number is allowed to + vary. If *numPos* is None, all numbers may vary. """ + # Turn the name into a SeqString + if not isinstance(name, SeqString): + name = SeqString(name) + if len(self._names)==0: return True else: return self._names[0].match(name, numPos) def append(self, name, obj=None): - """Add a file name to the group. + """Append a name/object to the end of the sequence. - name can be a SeqString object or a regular string. - The name is added unconditionally, so it's the callers responsibility - to make sure the file really belongs to this sequence. + *name* can be a :class:SeqString object or a regular string. + The name must match the names in the sequence, otherwise a + :exc:ValueError exception is thrown. - This is called internally when sequences are built! + *obj* can be any Python object that is stored alongside the name + (this is supposed to be the actual object that has the given name). + In any sequence, either all or none of the names must be associated + with an object. An attempt to append a name without an object to a + sequence that has objects will trigger a :exc:ValueError exception. + + Usually, you won't call this method manually to build a sequence + but instead use the :func:buildSequences() function which returns + initialized Sequence objects. """ + # Turn the name into a SeqString if not isinstance(name, SeqString): name = SeqString(name) + if not self.match(name): + placeholder,ranges = self.sequenceName() + raise ValueError("Cannot add '%s' to sequence %s. The name doesn't match the sequence."%(name, placeholder)) + if obj is not None: if self._objects is None: if len(self._names)==0: @@ -524,6 +575,8 @@ else: raise ValueError("objects must be given for all or none of the names") self._objects.append(obj) + elif self._objects is not None: + raise ValueError("objects must be given for all or none of the names") self._names.append(name) @@ -531,9 +584,9 @@ """Return the index of the sequence number. Returns the index of the number that has the most variation among its - values. If two numbers have the same amount of values, the last - number is used. - Returns None if there is no number at all. + values. If two number positions have the same variation, then the last + number is returned. + Returns None if there is no number at all. """ ranges = self.ranges() @@ -549,9 +602,9 @@ return seqNumIdx def ranges(self): - """Return a list of all the number ranges in the sequence. + """Returns a list of all the number ranges in the sequence. - The return value is a list of Range objects. There are as many + The return value is a list of :class:Range objects. There are as many ranges as there are separate numbers in the names. The ranges are given in the same order as the corresponding number appears in the names. @@ -562,15 +615,15 @@ def sequenceName(self): """Return a sequence placeholder and range strings. - Returns a tuple (placeholder, ranges) where placeholder is the + Returns a tuple (*placeholder*, *ranges*) where *placeholder* is the name of a member of the sequence where all numbers have been replaced - by '#' (=0-padded number with 4 digits) or one or more '@' (=padded - number with as many digits as there are '@' characters. Just a single - '@' represents an unpadded number). If the sequence contains inconsistent - padding, the number is replaced by '*'. + by '#' (0-padded number with 4 digits) or one or more '@' (padded + number with as many digits as there are '@' characters. Just a single + '@' represents an unpadded number). If the sequence contains inconsistent + padding, the number is replaced by '*'. The number is not replaced at all if there is only one single value among all file names anyway. - ranges is a list of strings where each string describes the range + *ranges* is a list of strings where each string describes the range of values of the corresponding number in the placeholder string. The returned information is meant to be displayed to the user as @@ -653,118 +706,45 @@ return str(res), rangeStrs -class Sequences: - """A collection of sequences. - """ - - def __init__(self, names=[], assumeFiles=False, getNameFunc=None): - """Constructor. - - names is a list of strings that will be grouped into sequences. - """ - # A list of FileSequence objects - self._sequences = [] - - # Create the sequences - self.setFiles(names, assumeFiles=assumeFiles, getNameFunc=getNameFunc) - def __str__(self): - return ""%len(self._sequences) - - def __len__(self): - return len(self._sequences) - - def __getitem__(self, key): - """Return the Sequence object with the given index. - """ - return self._sequences[key] - - # clear - def clear(self): - """Remove all sequences. - """ - self._sequences = [] - - # setFiles - def setFiles(self, names, numPos=None, assumeFiles=False, getNameFunc=None): - """Initialize the sequences given a flat list of names. - - names is a list of objects (usually strings) that are turned into - SeqString objects and grouped into sequences. - if assumeFiles is True, the input strings are assumed to be file - names. In this case, it will be ensured that files from different - directories are put into different sequences and any number occurring - in the directory part is "frozen" (turned into a string). - - getNameFunc can be a callable that gets called for every item in names. - The function has to return the actual name of that object. This can - be used if the input list contains object that are not strings but - some other (compound) objects. - """ - - self.clear() - - # Create the objects list which contains 2-tuples (seqString,obj). - # obj is the original object from the "names" list or None. - if getNameFunc is None: - objects = map(lambda name: (SeqString(name),None), names) - else: - objects = map(lambda obj: (SeqString(getNameFunc(obj)),obj), names) - # Sort the objects according to their seqString - # The order of the result is already so that members of the same - # sequence are together, we just don't know yet where a sequence ends - # and the next one begins. - objects.sort(key=lambda tup: tup[0]) - - # Build sequences... - currentSeq = Sequence() - currentPath = None - for name,obj in objects: - # Are we dealing with file names? Then freeze directory numbers... - if assumeFiles: - path,n = os.path.split(str(name)) - pathseq = SeqString(path) - # n: The number count in the path (these numbers have to be frozen) - n = pathseq.numCount() - for i in range(n): - name.replaceNum(i, name.getNumStr(i)) - - sequenceSplit = False - - # Check if the current name has a different structure or different - # text parts as the names in the current sequence. If so, we - # have to begin a new sequence - if not currentSeq.match(name, numPos): - sequenceSplit = True - - # If we are dealing with file names, then make sure files in - # different directories are put into separate sequences (even - # when the names have the same structure). - if assumeFiles: - # path has been set above where the directory numbers were frozen - if currentPath is not None and path!=currentPath: - sequenceSplit = True - currentPath = path - - # Do we have to begin a new sequence? - if sequenceSplit: - self._sequences.append(currentSeq) - currentSeq = Sequence() - - # Add the current name to the current sequence - currentSeq.append(name, obj) - - # Also store the last sequence generated (if it isn't empty) - if len(currentSeq)>0: - self._sequences.append(currentSeq) - - class Range: """Range class. - This class represents a sequence of integer values (frame numbers). + This class represents a sorted sequence of integer values (frame numbers). The sequence is composed of a number of sub-ranges which have a begin, - an optional end and a step number. + an optional end and an optional step number. If the end is omitted, + the sequence will be infinite. + + Examples: + + >>> list(Range("1,5,10")) + [1, 5, 10] + >>> list(Range("1-5")) + [1, 2, 3, 4, 5] + >>> list(Range("2-8x2")) + [2, 4, 6, 8] + >>> list(Range("1-3,10-13")) + [1, 2, 3, 10, 11, 12, 13] + + The range object supports the :func:len() operator, comparison operators, + the :keyword:in operator and iteration. Examples: + + >>> rng = Range("1-2,5") + >>> len(rng) + 3 + >>> for i in rng: print i + ... + 1 + 2 + 5 + >>> 3 in rng + False + >>> 5 in rng + True + >>> Range("1-3")==Range("1,2,3") + True + >>> Range("1-5")==Range("2-6") + False """ def __init__(self, rangeStr=None): @@ -802,11 +782,27 @@ endStr = str(end) rangeStrs.append("%s-%s%s"%(begin,endStr,stepStr)) - + return ",".join(rangeStrs) __repr__ = __str__ - + + def __eq__(self, other): + """Equality operator + """ + if not isinstance(other, Range): + return False + + return self._ranges==other._ranges + + def __ne__(self, other): + """Inequality operator + """ + if not isinstance(other, Range): + return True + + return self._ranges!=other._ranges + def __len__(self): """Return the number of values in the sequence. @@ -853,49 +849,54 @@ # Remove the deleted items (the ones that are None) currentValues = filter(lambda x: x is not None, currentValues) - + + def __contains__(self, val): + """Check if a value is inside the range. + + *val* is an integer that is checked against the range. The method + returns True when the value is part of the range. + """ + for begin,end,step in self._ranges: + if val>=begin and (end is None or val<=end) and (val-begin)%step==0: + return True + return False + def isInfinite(self): """Check if the range is infinite. + + Examples: + + >>> Range("1-5").isInfinite() + False + >>> Range("1-").isInfinite() + True """ for begin,end,step in self._ranges: if end is None: return True - return False - - def contains(self, val): - """Check if a value is inside the range. - - val is an integer that is checked against the range. The method - returns True when the value is part of the range. - """ - for begin,end,step in self._ranges: - if val>=begin and (end is None or val<=end) and (val-begin)%step==0: - return True return False def setRange(self, rangeStr): """Initialize the range object with a new range string. The range string may contain individual numbers or ranges separated by - comma. The individual ranges are specified by a begin, an end (inclusive) - and an optional step number. - This is the opposite function to compactRange(). + comma. The individual ranges are specified by a begin, an optional + end (inclusive) and an optional step number. Passing None is + equivalent to passing an empty string. - Examples: - - "1,5,10" -> [1,5,10] - "1-5" -> [1,2,3,4,5] - "2-8x2" -> [2,4,6,8] - "1-3,10-13" -> [1,2,3,10,11,12,13] + This is the opposite operation to e :func:compactRange() function. """ if rangeStr is None: rangeStr = "" + if type(rangeStr) is not str: + raise TypeError("The rangeStr argument must be a string") + reRange = re.compile(r"([0-9]+)(?:-([0-9]*)(?:x([0-9]+))?)?$") - self._ranges = [] + ranges = [] for rs in rangeStr.split(","): rs = rs.strip() if rs=="": @@ -921,18 +922,192 @@ # sequence (i.e. 1-10x2 -> 1-9x2) end -= (end-begin)%step if end is None or end>=begin: - self._ranges.append((begin,end,step)) + ranges.append((begin,end,step)) else: raise ValueError("Invalid range string: %s"%rs) - self._ranges.sort() + ranges = self._normalizeRanges(ranges) + self._ranges = ranges + def _normalizeRanges(self, ranges): + """Normalize the given ranges. + ranges is a list of range tuples (just like self._ranges). + Sorts the ranges, merges them if possible (1,2,3 -> 1-3) or + splits them up so that they don't overlap (2-20x2,11 -> 2-10x2,11,12-20x2). + Returns a new range list (the input list gets destroyed). + """ + if len(ranges)==0: + return [] + + ranges.sort() + + newRanges = [] + # The current range + rng = ranges.pop(0) + while len(ranges)>0: + # Get the next range + nextRng = ranges.pop(0) + + # Handle range overlaps + rngs = self._resolveRangeOverlap(rng, nextRng) + if rngs is not None: + rng = rngs[0] + # Only 1 range? Then nextRange was completely contained in rng, so get a new range + if len(rngs)>1: + # Continue with the adjusted ranges (insert them into the range + # list and sort again because the order may have changed) + ranges.extend(rngs[1:]) + ranges.sort() + continue + + # Merge the ranges if possible... + rng,nextRng = self._mergeRanges(rng, nextRng) + if nextRng is not None: + newRanges.append(rng) + rng = nextRng + + # Append the last range + newRanges.append(rng) + + # Final step that moves end values to the subsequent range if this + # makes the sub-ranges "nicer". + for i in range(len(newRanges)-1): + begin1,end1,step1 = newRanges[i] + begin2,end2,step2 = newRanges[i+1] + # Can the last value of the current range be moved into the subsequent + # range and the current range would then only be one single value? + # (Example: 1-5x4,6-10 -> 1,5-10) + if end1==begin2-step2 and begin1+step1==end1: + newRanges[i] = (begin1,begin1,1) + newRanges[i+1] = (end1,end2,step2) + + return newRanges + + def _resolveRangeOverlap(self, rng1, rng2): + """Resolve overlapping ranges. + + rng1 and rng2 are two adjacent ranges in sorted order (rng2 must + not be before rng1). + Returns a list of 1-3 ranges where the first range is guaranteed + to be non-overlapping. The other ranges are in sort order but may + still overlap (they will be handled in a subsequent iteration). + Returns None when rng1 and rng2 don't overlap at all. + + This is a helper method for _normalizeRanges(). + """ + begin1,end1,step1 = rng1 + begin2,end2,step2 = rng2 + + # No overlap? Then don't modify anything + if end1 is not None and begin2>end1: + return None + + # The ranges overlap... + + # Remove all initial values from rng2 that are also part of rng1. + # First check if begin2 is part of rng1 + if (begin2-begin1)%step1==0: + # Does rng2 use a step size that is a multiple of the step size of rng1? + if step2%step1==0: #step1==step2: + # Does rng2 completely lie within rng1? Then just ignore rng2 + if end1 is None or (end2 is not None and end2<=end1): + return [rng1] + else: + # Set the begin of rng2 to the first value behind the end of rng1 + n = int((end1-begin2)/step2)+1 + begin2 += n*step2 + # Different steps, so only the first value is identical + else: + # Is rng2 just one single value? Then we can ignore rng2 + # (because this value is also part of rng1) + if begin2==end2: + return [rng1] + else: + begin2 += step2 + + # If the ranges don't overlap anymore, then we are done. + if end1 is not None and begin2>end1: + return [rng1,(begin2,end2,step2)] + + + # At this point, it is guaranteed that... + # - ...rng1 and rng2 don't begin with the same value (i.e. begin1 adjust rng1 so that it only contains the remaining range + n = int((begin2-begin1-1)/step1) + e1 = begin1+n*step1 + res.append((begin1,e1,step1)) + begin1 = e1+step1 + + # begin1 is now greater than begin2 (they can't be equal because we know + # that begin2 is not part of the initial rng1) + + res.append((begin2,end2,step2)) + res.append((begin1,end1,step1)) + + # res now contains 3 ranges. The first one is guaranteed to be unique + # and doesn't overlap anymore. The other two may still overlap but + # this is dealt with in a subsequent iteration. + + return res + + def _mergeRanges(self, rng1, rng2): + """Merge two ranges if possible. + + Returns the new ranges. The second range may become None if it was + entirely consumed by the first range. + + The input ranges must be sorted (i.e. rng2 must not be *before* rng1). + + This is a helper method for _normalizeRanges(). + """ + begin1,end1,step1 = rng1 + begin2,end2,step2 = rng2 + + # If range1 is just a single value, then we can always merge at least + # the first value of range2 (as we are free to change the step). + if begin1==end1: + step1 = begin2-end1 + + # If range2 does not start right behind range1, then there is nothing to merge + if end1 is None or begin2!=end1+step1: + return rng1,rng2 + + # Can the entire range2 be merged into range1? (this is the case if + # the step size is identical or range2 is just a single value anyway) + if step1==step2 or begin2==end2: + return (begin1,end2,step1),None + # Only put the first value of range2 into range1 + else: + return (begin1,begin2,step1), (begin2+step2, end2, step2) + + class SeqTemplate: - """Sequence template class. + """Sequence name template class. - An instance of this class represents a template string to create numbered - sequences. + An instance of this class represents a template string that may contain + patterns that will be substituted by numbers. + This can be used to generate the individual names for an output sequence. + + Example: + + >>> tmpl = SeqTemplate("foo#.tif") + >>> tmpl([17]) + 'foo0017.tif' + >>> tmpl=SeqTemplate("foo@@_#.tif") + >>> tmpl([2,17]) + 'foo02_0017.tif' + >>> tmpl=SeqTemplate("foo@@[2]_#[1].tif") + >>> tmpl([2,17]) + 'foo17_0002.tif' + >>> tmpl=SeqTemplate("foo{2*#+1}.tif") + >>> tmpl([5]) + 'foo0011.tif' """ def __init__(self, template): @@ -965,13 +1140,15 @@ """Return a string that uses the given input numbers. The substitution patterns in the template string are replaced by - the given numbers. values must be a list of objects that can be + the given numbers. *values* must be a list of objects that can be turned into integers. - It is the callers responsibility to make sure that values contains + It is the callers responsibility to make sure that *values* contains enough numbers. - If any number expression fails, a ValueError exception is thrown - (this is also the case when the expressions refers to a value in + If any number expression fails, a :exc:ValueError exception is thrown + (this is also the case when an expression refers to a value in the input list that is not available). + + Calling this method is equivalent to using the object as a callable. """ # Make sure we have integers values = [int(v) for v in values] @@ -991,13 +1168,24 @@ def expressionIndices(self, inputSize): """Return the indices of the source values that the number expressions refer to. - inputSize is the length of the value sequence that will get passed - to substitute(). This is used to resolve negative indices. The + *inputSize* is the length of the value sequence that will get passed + to :meth:substitute(). This is used to resolve negative indices. The result may still contain negative indices if any index in the expressions is out of range. The order of the values in the list is the same order as the expressions appear in the template. The return value can be used to check if an expression would produce - an IndexError exception. + an :exc:IndexError exception. + + Example: + + >>> t=SeqTemplate("foo#_#") + >>> t.expressionIndices(2) + [0, 1] + >>> t=SeqTemplate("foo#[-1]_#[1]") + >>> t.expressionIndices(2) + [1, 0] + >>> t.expressionIndices(3) + [2, 0] """ res = [] for i in self._exprIndices: @@ -1132,15 +1320,40 @@ output file sequence based on an input sequence but where the numbers in the output sequence may be different than the numbers in the input sequence. For example, the class is used by the sequence utilities (seqmv, seqcp, - seqrm, seqln). + seqrm). - An OutputNameGenerator has one public attribute called numberMergeFlag - which is True when the output name pattern ended in a digit but didn't + An :class:OutputNameGenerator has one public attribute called :attr:numberMergeFlag + which is True when the output name pattern ended in a digit but didn't contain any number pattern. In this case, the class will append a 4-padded number but because the name already ended in a digit, the combination of the pattern and the number results in a larger number which is not necessarily what the user intended. The flag can be used by an application to check whether it should ask the user for confirmation. + + Example: + + >>> seqs = buildSequences(["spam1_1.tif", "spam1_2.tif", "spam1_5.tif"]) + >>> + >>> for src,dst in OutputNameGenerator(seqs, "foo"): + ... print src,"->",dst + ... + spam1_1.tif -> foo0001.tif + spam1_2.tif -> foo0002.tif + spam1_5.tif -> foo0005.tif + >>> + >>> for src,dst in OutputNameGenerator(seqs, "foo@...#.tif", dstRange=Range("10-")): + ... print src,"->",dst + ... + spam1_1.tif -> foo1_0010.tif + spam1_2.tif -> foo1_0011.tif + spam1_5.tif -> foo1_0012.tif + >>> + >>> for src,dst in OutputNameGenerator(seqs, "foo_#[2]_{@[1]+2}.tif"): + ... print src,"->",dst + ... + spam1_1.tif -> foo_0001_3.tif + spam1_2.tif -> foo_0002_3.tif + spam1_5.tif -> foo_0005_3.tif """ def __init__(self, srcSequences, dstName, srcRanges=None, dstRange=None, keepExt=True, @@ -1238,6 +1451,8 @@ for srcSeq in self._srcSequences: self._outputNameSpec(srcSeq, dstName, dstRangeIter is not None) + def __iter__(self): + return self.iterNames() def iterNames(self): """Iterate over input/output name pairs. @@ -1246,6 +1461,8 @@ name from the input sequences and dstName is the generated output name (as specified by the output pattern and additional arguments that were passed to the constructor). + + This is equivalent to iterating directly over the object. """ # Iterate over all input sequences @@ -1280,6 +1497,27 @@ res = self._outputNameSpec(srcSequence, dstName, dstRangeIter is not None) dstTemplate, numIdxs, seqNumIdx = res + # Check what indices are used by the expressions (the result may not be + # accurate when negative numbers are used because the integer we pass + # to expressionIndices() may not be the correct one, but we are only + # really interested in the simpler case were no explicit indices have + # been provided anyway). + ei = dstTemplate.expressionIndices(len(numIdxs)) + # Adjust the index of the main sequence number if it is not in the + # list of used indices. Otherwise providing a destination range would + # be useless because it would affect an unused number. + # This can happen when an input sequence has at least two varying numbers + # and the output sequence has only one number pattern and a destination + # range has been specified. + # Example: "spam1_1", "spam1_2", "spam2_5", "spam2_6" -> "foo#" (2-) + # The main sequence number will be the second one, but the pattern + # in the output name would refer to the first number, so the destination + # range would have no effect and the output would be "foo1", "foo1", + # "foo2", "foo2". The following if sets the main sequence number to be + # the first one and then everything is fine again. + if seqNumIdx not in ei: + seqNumIdx = max(ei) + srcIter = iter(srcSequence) # Assign output names to the input names... @@ -1310,7 +1548,7 @@ nums = map(lambda i: allNums[i], numIdxs) # Only queue this file when it is part of the source range - if len(nums)==0 or srcRange.contains(nums[seqNumIdx]): + if len(nums)==0 or (nums[seqNumIdx] in srcRange): # If a destination range was specified then replace the # main file number with the next number in the range, otherwise # the number from the input file is used @@ -1319,7 +1557,7 @@ nums[seqNumIdx] = dstRangeIter.next() except StopIteration: break - # Create the file names and add them to the list + # Create the file names dstName = dstTemplate.substitute(nums) if keepExt and os.path.splitext(dstName)[1]!=ext: dstName += ext @@ -1392,7 +1630,7 @@ for i,rng in enumerate(ranges): if len(rng)>1: numIdxs.append(i) - # Do we have too little patterns? (and the user did not specify any + # Do we have too few patterns? (and the user did not specify any # index explicitly?) # If so, throw an error because it's not clear which number should be # mapped to which pattern. @@ -1427,7 +1665,16 @@ """Base class for move/copy/link. """ - def __init__(self, srcSequences, dstName, srcRanges=None, dstRange=None, keepExt=True, enforceDstRange=False, verbose=False): + def __init__(self, srcSequences, dstName, srcRanges=None, dstRange=None, + keepExt=True, enforceDstRange=False, verbose=False, + resolveSrcLinks=False): + """Constructor. + + See the derived classes for documentation on most arguments. + + resolveSrcLinks: If true, the source file names will be replaced by + their real paths. + """ ong = OutputNameGenerator(srcSequences, dstName, srcRanges = srcRanges, @@ -1441,8 +1688,10 @@ # Create the file table fileTab = [] for uiSrc,uiDst in ong.iterNames(): - src = os.path.realpath(uiSrc) - dst = os.path.realpath(uiDst) + src = os.path.abspath(uiSrc) + dst = os.path.abspath(uiDst) + if resolveSrcLinks: + src = os.path.realpath(uiSrc) fileTab.append((src,dst,uiSrc,uiDst)) # Resolve internal collisions @@ -1454,11 +1703,11 @@ def mergesNumbers(self): """Check if a trailing number on the output sequence and a file number would get merged. - This method returns True when the base output sequence name ends in + This method returns True when the base output sequence name ends in a number and a sequence number would be appended as well which results in a new number (for example, writing a sequence with the base name - out2 can produce output files out20001, out20002, ... which may not - be what the user intended). The result of this call can be used to + out2 can produce output files out20001, out20002, ... which + may not be what the user intended). The result of this call can be used to check if the application should ask the user for confirmation. """ return self._mergesNumbers @@ -1468,7 +1717,7 @@ Only iterates over the files that are not part of the input sequence. The returned files are those that would get overwritten when the - move operation would be carried out. + operation would be carried out. This can be used to check if the user should be asked for confirmation. """ srcDict = {} @@ -1485,22 +1734,28 @@ def sequences(self): """Iterate over the input/output sequences. - Yields tuples (srcSeq, dstSeq) where each item is a Sequence() + Yields tuples (*srcSeq*, *dstSeq*) where each item is a :class:Sequence object. The result can be used to show an overview of what the - move operation will do. + operation will do. """ # Print the final source and destination sequences (just for user info) - seqs = Sequences(self._fileTab, getNameFunc=lambda t:t[2]) + + # We use _buildSequences() instead of buildSequences() so that the + # fileTab doesn't get sorted again (it is already sorted). This + # ensures that the sequences are yielded in the same order in which + # they will get processed. + objects = map(lambda obj: (SeqString(obj[2]),obj), self._fileTab) + seqs = _buildSequences(objects) for srcSeq in seqs: dstFiles = map(lambda t: t[3], srcSeq) - dstSeq = Sequences(dstFiles)[0] + dstSeq = buildSequences(dstFiles)[0] yield srcSeq, dstSeq def dryRun(self, outStream=None): """Print what would get done when run() was called. - outStream is an object with a write() method that will receive - the text. If None is passed, sys.stdout is used. + *outStream* is an object with a :meth:write() method that will + receive the text. If None is passed, sys.stdout is used. """ if outStream is None: outStream = sys.stdout @@ -1512,8 +1767,9 @@ def run(self, outStream=None): """Do the operation. - outStream is an object with a write() and flush() method that will receive - the text (only in verbose mode). If None is passed, sys.stdout is used. + *outStream* is an object with a :meth:write() and :meth:flush() + method that will receive the text (only in verbose mode). If None + is passed, sys.stdout is used. """ if outStream is None: outStream = sys.stdout @@ -1679,7 +1935,8 @@ """This class copies one or more sequences of files. """ - def __init__(self, srcSequences, dstName, srcRanges=None, dstRange=None, keepExt=True, verbose=False): + def __init__(self, srcSequences, dstName, srcRanges=None, dstRange=None, + keepExt=True, verbose=False, resolveSrcLinks=False): """Constructor. srcSequences is a list of Sequence objects that contain the source @@ -1718,8 +1975,13 @@ The verbose flag determines whether each file is printed during the actual operation. + + resolveSrcLinks determines whether source links are resolved before + processing the sequence. """ - _SequenceProcessor.__init__(self, srcSequences, dstName, srcRanges, dstRange, keepExt, enforceDstRange=True, verbose=verbose) + _SequenceProcessor.__init__(self, srcSequences, dstName, srcRanges, dstRange, + keepExt, enforceDstRange=True, verbose=verbose, + resolveSrcLinks=resolveSrcLinks) def _fileOperation(self, src, dst): """Do the copy operation. @@ -1788,18 +2050,111 @@ os.symlink(src, dst) +def buildSequences(names, numPos=None, assumeFiles=False, nameFunc=None): + """Create sorted sequences from a list of names/objects. + + *names* is a list of objects (usually strings) that are grouped into sequences. + If *assumeFiles* is True, the input strings are assumed to be file + names. In this case, it will be ensured that files from different + directories are put into different sequences and any number occurring + in the directory part is "frozen" (turned into a string). + + *numPos* can be set to a number index which defines the position of the + numbers that are allowed to vary per sequence. If not given, all numbers + may vary (for example, if you want the files clip1_#.tif to be a different + sequence than clip2_#.tif you have to set numPos to 1 or -1). + + *nameFunc* can be a callable that gets called for every item in *names*. + The function has to return the actual name of that object. This can + be used if the input list contains objects that are not strings but + some other (compound) objects. + + Returns a list of :class:Sequence objects. + The sequences and the files within the sequences are sorted. + """ + # Create the objects list which contains 2-tuples (seqString,obj). + # obj is the original object from the "names" list or None. + if nameFunc is None: + objects = map(lambda name: (SeqString(name),None), names) + else: + objects = map(lambda obj: (SeqString(nameFunc(obj)),obj), names) + # Sort the objects according to their seqString + # The order of the result is already so that members of the same + # sequence are together, we just don't know yet where a sequence ends + # and the next one begins. + objects.sort(key=lambda tup: tup[0]) + + return _buildSequences(objects, numPos, assumeFiles) + +def _buildSequences(objects, numPos=None, assumeFiles=False): + """Helper function for buildSequences(). + + objects is a sorted list of (name,obj) tuples. + """ + res = [] + + # Build sequences... + currentSeq = Sequence() + currentPath = None + for name,obj in objects: + # Are we dealing with file names? Then freeze directory numbers... + if assumeFiles: + path,n = os.path.split(str(name)) + pathseq = SeqString(path) + # n: The number count in the path (these numbers have to be frozen) + n = pathseq.numCount() + for i in range(n): + name.replaceNum(i, name.getNumStr(i)) + + sequenceSplit = False + + # Check if the current name has a different structure or different + # text parts as the names in the current sequence. If so, we + # have to begin a new sequence + if not currentSeq.match(name, numPos): + sequenceSplit = True + + # If we are dealing with file names, then make sure files in + # different directories are put into separate sequences (even + # when the names have the same structure). + if assumeFiles: + # path has been set above where the directory numbers were frozen + if currentPath is not None and path!=currentPath: + sequenceSplit = True + currentPath = path + + # Do we have to begin a new sequence? + if sequenceSplit: + res.append(currentSeq) + currentSeq = Sequence() + + # Add the current name to the current sequence + currentSeq.append(name, obj) + + # Also store the last sequence generated (if it isn't empty) + if len(currentSeq)>0: + res.append(currentSeq) + + return res + + def compactRange(values): """Build the range string that lists all values in the given list in a compacted form. - values is a list of integers (may contain duplicate values and doesn't have + *values* is a list of integers (may contain duplicate values and does not have to be sorted). The return value is a string that lists all values (sorted) - in a compacted form (using the same syntax that Shake accepts as time values). - The returned range string can be passed to a Range object to create the - expanded integer sequence again. + in a compacted form. + The returned range string can be passed into a :class:Range object to create + the expanded integer sequence again. - Example: [1,2,3,4,5,6] -> "1-6" - [2,4,6,8] -> "2-8x2" - [1,2,3,10,11,12] -> "1-3,10-12" + Examples: + + >>> compactRange([1,2,3,4,5,6]) + '1-6' + >>> compactRange([2,4,6,8]) + '2-8x2' + >>> compactRange([1,2,3,12,11,10]) + '1-3,10-12' """ if len(values)==0: return "" @@ -1870,12 +2225,14 @@ return ",".join(rs) def glob(name): - """Create file sequences. + """Create file sequences from a name pattern. - name is a file pattern that will get a '*' appended. The pattern is then - passed to the regular glob() function to obtain a list of files which - are then grouped into sequences. - Returns a Sequences objects that contains all file sequences found. + *name* is a file pattern that will get a '*' appended. The pattern is then + passed to the regular :func:glob() function from the standard :mod:glob + module to obtain a list of files which are then grouped into sequences. + + Returns a list of :class:Sequence objects. + The sequences and the files within the sequences are sorted. """ globpattern = name if not globpattern.endswith("*"): @@ -1926,7 +2283,7 @@ # Remove files that don't have any number in their name (without ext) fileNames = filter(lambda n: SeqString(os.path.splitext(n)[0]).numCount()>0, fileNames) - return Sequences(fileNames, assumeFiles=True) + return buildSequences(fileNames, assumeFiles=True) # The following function is obsolete and replaced by the SeqTemplate class. Added: cgkit/trunk/unittests/test_sequence.py =================================================================== --- cgkit/trunk/unittests/test_sequence.py (rev 0) +++ cgkit/trunk/unittests/test_sequence.py 2009-08-08 09:09:38 UTC (rev 330) @@ -0,0 +1,733 @@ +# Test the sequence module + +import unittest +import os, os.path +import glob as globmod +from cgkit.sequence import * + +class TestSeqString(unittest.TestCase): + """Test the SeqString class. + """ + + def testComparison(self): + """Test the comparison operator. + """ + s1 = SeqString("spam004") + s2 = SeqString("spam2") + s3 = SeqString("spam4") + self.assertEqual(1, cmp(s1, s2)) + self.assertEqual(-1, cmp(s2, s1)) + self.assertEqual(0, cmp(s1, s3)) + + s1 = SeqString("spam004_018") + s2 = SeqString("spam2_431") + s3 = SeqString("spam4_18") + self.assertEqual(1, cmp(s1, s2)) + self.assertEqual(-1, cmp(s2, s1)) + self.assertEqual(0, cmp(s1, s3)) + + self.assertEqual("spam004_018", s1) + self.assertEqual(True, "spam004_018"==s1) + self.assertEqual(True, "spam04_18"==s1) + self.assertEqual(True, "spam04_19"!=s1) + + s = SeqString("spam1_1") + self.assertEqual(True, s=="spam1_1") + s.replaceNum(0, "1") + self.assertEqual(True, s=="spam1_1") + + def testMatch(self): + """Test the match method. + """ + s = SeqString("foo") + self.assertEqual(True, s.match(SeqString("foo"))) + self.assertEqual(False, s.match(SeqString("Foo"))) + self.assertEqual(False, s.match(SeqString("bar"))) + self.assertEqual(False, s.match(SeqString("foo10"))) + self.assertEqual(False, s.match(SeqString(""))) + + s = SeqString("spam1_bla05") + self.assertEqual(True, s.match(SeqString("spam05_bla1000"))) + self.assertEqual(False, s.match(SeqString("spam05_bla1000x"))) + self.assertEqual(False, s.match(SeqString("spam_bla1000x"))) + self.assertEqual(False, s.match(SeqString("spam2_bla07"),-1)) + self.assertEqual(True, s.match(SeqString("spam1_bla07"),-1)) + self.assertEqual(False, s.match(SeqString("spam1_bla07"),0)) + self.assertEqual(False, s.match(SeqString("spam1_bla5"),0)) + self.assertEqual(True, s.match(SeqString("spam2_bla05"),0)) + self.assertEqual(False, s.match(SeqString("spam2_bla05"),3)) + self.assertEqual(True, s.match(SeqString("spam1_bla05"),3)) + self.assertEqual(False, s.match(SeqString("spam2_bla04"),-3)) + + self.assertRaises(TypeError, lambda: s.match("spam2_bla04")) + + def testGroupRepr(self): + """Test the groupRepr() method. + """ + self.assertEqual("foo", SeqString("foo").groupRepr()) + self.assertEqual("foo*.tif", SeqString("foo100.tif").groupRepr()) + self.assertEqual("spam#_#", SeqString("spam3_0983").groupRepr("#")) + self.assertEqual("spam_", SeqString("spam3_0983").groupRepr("")) + + def testNumCount(self): + """Test the numCount() method. + """ + self.assertEqual(1, SeqString("anim01").numCount()) + self.assertEqual(2, SeqString("anim2_01").numCount()) + self.assertEqual(0, SeqString("anim").numCount()) + + def testGetNum(self): + """Test the getNum() method. + """ + self.assertEqual(17, SeqString("spam17_042_008").getNum(0)) + self.assertEqual(42, SeqString("spam17_042_008").getNum(1)) + self.assertEqual(8, SeqString("spam17_042_008").getNum(2)) + self.assertEqual(17, SeqString("spam17_042_008").getNum(-3)) + self.assertEqual(42, SeqString("spam17_042_008").getNum(-2)) + self.assertEqual(8, SeqString("spam17_042_008").getNum(-1)) + self.assertRaises(IndexError, lambda: SeqString("spam17_042_008").getNum(3)) + self.assertRaises(IndexError, lambda: SeqString("spam17_042_008").getNum(-4)) + self.assertRaises(IndexError, lambda: SeqString("spam").getNum(0)) + + def testGetNumStr(self): + """Test the getNumStr() method. + """ + self.assertEqual("17", SeqString("spam17_042_008").getNumStr(0)) + self.assertEqual("042", SeqString("spam17_042_008").getNumStr(1)) + self.assertEqual("008", SeqString("spam17_042_008").getNumStr(2)) + self.assertEqual("17", SeqString("spam17_042_008").getNumStr(-3)) + self.assertEqual("042", SeqString("spam17_042_008").getNumStr(-2)) + self.assertEqual("008", SeqString("spam17_042_008").getNumStr(-1)) + self.assertRaises(IndexError, lambda: SeqString("spam17_042_008").getNumStr(3)) + self.assertRaises(IndexError, lambda: SeqString("spam17_042_008").getNumStr(-4)) + self.assertRaises(IndexError, lambda: SeqString("spam").getNumStr(0)) + + def testGetNums(self): + """Test the getNums() method. + """ + self.assertEqual([17,42,8], SeqString("spam17_042_008").getNums()) + self.assertEqual([], SeqString("spam").getNums()) + + def testSetNum(self): + """Test the setNum() method. + """ + s = SeqString("spam1_012") + s.setNum(0, 2) + self.assertEqual("spam2_012", str(s)) + s.setNum(1, 7) + self.assertEqual("spam2_007", str(s)) + s.setNum(-1, 4) + self.assertEqual("spam2_004", str(s)) + s.setNum(-2, 3) + self.assertEqual("spam3_004", str(s)) + s.setNum(-1, 9876) + self.assertEqual("spam3_9876", str(s)) + s.setNum(-1, 1) + self.assertEqual("spam3_001", str(s)) + s.setNum(-1, -2) + self.assertEqual("spam3_-02", str(s)) + self.assertEqual(-2, s.getNum(-1)) + s.setNum(-1, 7.6) + self.assertEqual("spam3_007", str(s)) + s.setNum(0, 2, width=4) + self.assertEqual("spam0002_007", str(s)) + + self.assertRaises(IndexError, lambda: s.setNum(2, 5)) + self.assertRaises(IndexError, lambda: s.setNum(-3, 5)) + + def testSetNums(self): + """Test the setNums() method. + """ + s = SeqString("spam1_012") + s.setNums([3,4]) + self.assertEqual("spam3_004", str(s)) + s.setNums([2]) + self.assertEqual("spam2_004", str(s)) + + self.assertRaises(IndexError, lambda: s.setNums([1,2,3])) + + def testGetNumWidth(self): + """Test the getNumWidth() and getNumWidths() method. + """ + self.assertEqual(2, SeqString("spam17_0042").getNumWidth(0)) + self.assertEqual(4, SeqString("spam17_0042").getNumWidth(1)) + self.assertEqual(2, SeqString("spam17_0042").getNumWidth(-2)) + self.assertEqual(4, SeqString("spam17_0042").getNumWidth(-1)) + self.assertRaises(IndexError, lambda: SeqString("spam17_042").getNumWidth(2)) + self.assertRaises(IndexError, lambda: SeqString("spam17_042").getNumWidth(-3)) + self.assertRaises(IndexError, lambda: SeqString("spam").getNumWidth(0)) + + self.assertEqual([2,4], SeqString("spam17_0042").getNumWidths()) + + def testSetNumWidth(self): + """Test the setNumWidth() and setNumWidths() method. + """ + s = SeqString("spam1_004") + s.setNumWidth(0, 2) + self.assertEqual("spam01_004", str(s)) + s.setNumWidth(0, 0) + self.assertEqual("spam1_004", str(s)) + s.setNumWidth(1, 4) + self.assertEqual("spam1_0004", str(s)) + s.setNumWidth(-1, 1) + self.assertEqual("spam1_4", str(s)) + s.setNumWidth(-2, 2) + self.assertEqual("spam01_4", str(s)) + + s.setNumWidths([3,4]) + self.assertEqual("spam001_0004", str(s)) + self.assertRaises(IndexError, lambda: s.setNumWidths([1,2,3])) + + def testDeleteNum(self): + """Test the deleteNum() method. + """ + s = SeqString("spam1_017_0003") + s.deleteNum(1) + self.assertEqual("spam1__0003", str(s)) + self.assertEqual(2, s.numCount()) + s = SeqString("spam1_017_0003") + s.deleteNum(-3) + self.assertEqual("spam_017_0003", str(s)) + self.assertRaises(IndexError, lambda: s.deleteNum(3)) + self.assertRaises(IndexError, lambda: s.deleteNum(-4)) + + def testReplaceNum(self): + """Test the replaceNum() method. + """ + s = SeqString("spam1_017_0003") + s.replaceNum(2, "foo") + self.assertEqual("spam1_017_foo", str(s)) + self.assertEqual(2, s.numCount()) + s.replaceNum(-2, "*") + self.assertEqual("spam*_017_foo", str(s)) + self.assertEqual(1, s.numCount()) + s.replaceNum(0, "#") + self.assertEqual("spam*_#_foo", str(s)) + self.assertEqual(0, s.numCount()) + self.assertRaises(IndexError, lambda: s.deleteNum(0)) + + s = SeqString("03") + s.replaceNum(0, "foo") + self.assertEqual("foo", str(s)) + self.assertEqual(0, s.numCount()) + + s = SeqString("03") + s.replaceNum(0, "") + self.assertEqual("", str(s)) + self.assertEqual(0, s.numCount()) + + def testReplaceStr(self): + """Test the replaceStr() method. + """ + s = SeqString("spam1_017_0003") + s.replaceStr(0, "foo") + self.assertEqual("foo1_017_0003", str(s)) + self.assertEqual(3, s.numCount()) + s.replaceStr(2, "--") + self.assertEqual("foo1_017--0003", str(s)) + s.replaceStr(3, ".tif") + self.assertEqual("foo1_017--0003.tif", str(s)) + + s = SeqString("spam1_017_0003") + s.replaceStr(-4, "foo") + self.assertEqual("foo1_017_0003", str(s)) + self.assertEqual(3, s.numCount()) + s.replaceStr(-2, "--") + self.assertEqual("foo1_017--0003", str(s)) + s.replaceStr(-1, ".tif") + self.assertEqual("foo1_017--0003.tif", str(s)) + + self.assertRaises(IndexError, lambda: s.replaceStr(4, "#")) + + s = SeqString("017_0003") + self.assertEqual(2, s.numCount()) + s.replaceStr(0, "spam") + self.assertEqual("spam017_0003", str(s)) + self.assertEqual(2, s.numCount()) + + s = SeqString("2") + s.replaceStr(0, "spam") + s.replaceStr(1, "eggs") + self.assertEqual("spam2eggs", str(s)) + + s = SeqString("") + s.replaceStr(0, "spam") + self.assertEqual("spam", str(s)) + self.assertRaises(IndexError, lambda: s.replaceStr(1, "foo")) + + +class TestRange(unittest.TestCase): + """Test the Range object. + """ + + def testRange(self): + r = Range() + self.assertEqual(0, len(r)) + self.assertEqual([], list(r)) + self.assertEqual("", str(r)) + self.assertEqual(False, r.isInfinite()) + + r = Range("2-10") + self.assertEqual(9, len(r)) + self.assertEqual([2,3,4,5,6,7,8,9,10], list(r)) + self.assertEqual("2-10", str(r)) + self.assertEqual(False, r.isInfinite()) + + r = Range("1,9-9, 3 ,5-7") + self.assertEqual(6, len(r)) + self.assertEqual([1,3,5,6,7,9], list(r)) + self.assertEqual("1-5x2,6-7,9", str(r)) + for i in range(15): + self.assertEqual(i in [1,3,5,6,7,9], i in r) + + r = Range("2-10x2") + self.assertEqual(5, len(r)) + self.assertEqual([2,4,6,8,10], list(r)) + self.assertEqual("2-10x2", str(r)) + self.assertEqual(False, r.isInfinite()) + + r = Range("6-3") + self.assertEqual(0, len(r)) + self.assertEqual([], list(r)) + self.assertEqual("", str(r)) + + r = Range("6-") + self.assertEqual(True, r.isInfinite()) + self.assertRaises(ValueError, lambda: len(r)) + it = iter(r) + self.assertEqual(6, it.next()) + self.assertEqual(7, it.next()) + self.assertEqual(8, it.next()) + self.assertEqual("6-", str(r)) + self.assertEqual(False, 5 in r) + self.assertEqual(True, 6 in r) + self.assertEqual(True, 9867324 in r) + for i in range(15): + self.assertEqual(i>5, i in r) + + r.setRange("1-3") + self.assertEqual(False, r.isInfinite()) + self.assertEqual(3, len(r)) + self.assertEqual([1,2,3], list(r)) + + self.assertRaises(ValueError, lambda: r.setRange("spam")) + + def testNormalization(self): + """Test the normalization. + """ + self.assertEqual("2-10", str(Range("2-8x2,3-10"))) + self.assertEqual("1-10", str(Range("1-5,6-10"))) + self.assertEqual("1-5x2,6-10", str(Range("1-9x2,5-10"))) + self.assertEqual("1-9x2", str(Range("1-9x2,5"))) + self.assertEqual("1-11x2", str(Range("1-9x2,5-11x2"))) + self.assertEqual("1-5x2,6,7,8-10,12", str(Range("1-9x2,6-12x2"))) + self.assertEqual("1-4,6", str(Range("1-4,3-6x3"))) + self.assertEqual("1-7x3,8-12", str(Range("1-7x3,7-12"))) + self.assertEqual("4,7-12", str(Range("4-7x3,7-12"))) + self.assertEqual("1-3x2", str(Range("1,3"))) + self.assertEqual("10-16x2,17,18-x2", str(Range("10-x2,17"))) + self.assertEqual("10-20x2,21-", str(Range("10-x2,21-"))) + + def testComparison(self): + """Test comparing range objects. + """ + r1 = Range("1-5,8") + r2 = Range("8,1-5") + r3 = Range("1-5,7") + r4 = Range("8,1-4,5") + + self.assertEqual(True, r1==r2) + self.assertEqual(False, r1==r3) + self.assertEqual(True, r1==r4) + self.assertEqual(False, r1!=r2) + self.assertEqual(True, r1!=r3) + self.assertEqual(False, r1!=r4) + self.assertEqual(False, r1==None) + self.assertEqual(True, r1!=None) + self.assertEqual(False, r1==5) + self.assertEqual(True, r1!=5) + + +class TestSequence(unittest.TestCase): + """Test the Sequence class. + """ + + def testSequence(self): + seq = Sequence() + self.assertEqual(0, len(seq)) + self.assertEqual([], list(seq)) + self.assertEqual(None, seq.sequenceNumberIndex()) + + seq.append("spam1") + self.assertEqual(1, len(seq)) + self.assertEqual(["spam1"], list(seq)) + self.assertEqual(True, isinstance(seq[0], SeqString)) + self.assertEqual(("spam1", []), seq.sequenceName()) + + seq.append("spam3") + self.assertEqual(2, len(seq)) + self.assertEqual(["spam1", "spam3"], list(seq)) + self.assertEqual(("spam@", ["1,3"]), seq.sequenceName()) + + seq.append("spam2") + self.assertEqual(3, len(seq)) + self.assertEqual(["spam1", "spam3", "spam2"], list(seq)) + self.assertEqual(("spam@", ["1-3"]), seq.sequenceName()) + + seq.append("spam3") + self.assertEqual(4, len(seq)) + self.assertEqual(["spam1", "spam3", "spam2", "spam3"], list(seq)) + self.assertEqual(["spam1", "spam3", "spam2", "spam3"], list(seq.iterNames())) + self.assertEqual(["spam1", "spam3", "spam2", "spam3"], list(seq.iterObjects())) + self.assertEqual(("spam@", ["1-3"]), seq.sequenceName()) + + self.assertEqual(SeqString("spam1"), seq[0]) + self.assertEqual(SeqString("spam3"), seq[1]) + self.assertEqual(SeqString("spam2"), seq[2]) + self.assertEqual(SeqString("spam3"), seq[3]) + self.assertRaises(IndexError, lambda: seq[4]) + self.assertEqual(SeqString("spam3"), seq[-1]) + self.assertEqual(SeqString("spam2"), seq[-2]) + self.assertEqual(SeqString("spam3"), seq[-3]) + self.assertEqual(SeqString("spam1"), seq[-4]) + self.assertRaises(IndexError, lambda: seq[-5]) + + self.assertEqual([Range("1-3")], seq.ranges()) + self.assertEqual(0, seq.sequenceNumberIndex()) + + self.assertRaises(ValueError, lambda: seq.append("spam4", obj=4)) + + self.assertRaises(ValueError, lambda: seq.append("spam3_1")) + + self.assertEqual(True, seq.match("spam02")) + self.assertEqual(False, seq.match("foo02")) + + def testMultiSequence(self): + """Test a sequence that has more than 1 number in its name. + """ + seq = Sequence() + seq.append("clip1_02") + self.assertEqual(1, len(seq)) + self.assertEqual(["clip1_02"], list(seq)) + self.assertEqual(("clip1_02", []), seq.sequenceName()) + self.assertEqual(1, seq.sequenceNumberIndex()) + + seq.append("clip1_03") + self.assertEqual(("clip1_@@", ["2-3"]), seq.sequenceName()) + self.assertEqual(1, seq.sequenceNumberIndex()) + + seq = Sequence() + seq.append("clip1_02") + seq.append("clip2_02") + self.assertEqual(("clip@...", ["1-2"]), seq.sequenceName()) + self.assertEqual(0, seq.sequenceNumberIndex()) + + def testSequenceObjects(self): + """Test inserting objects instead of just names. + """ + seq = Sequence() + seq.append("spam1", obj=1) + seq.append("spam2", obj=2) + self.assertRaises(ValueError, lambda: seq.append("spam3")) + self.assertEqual([1,2], list(seq)) + self.assertEqual(["spam1", "spam2"], list(seq.iterNames())) + self.assertEqual([1,2], list(seq.iterObjects())) + + +class TestBuildSequences(unittest.TestCase): + """Test the buildSequences() function. + """ + + def testBuildSequences(self): + seqs = buildSequences([]) + self.assertEqual([], seqs) + + seqs = buildSequences(["spam1", "spam3", "spam2"]) + self.assertEqual(1, len(seqs)) + self.assertEqual(["spam1", "spam2", "spam3"], list(seqs[0])) + + seqs = buildSequences(["clip1_1", "clip2_1", "clip1_2", "clip2_2"]) + self.assertEqual(1, len(seqs)) + self.assertEqual(["clip1_1", "clip1_2", "clip2_1", "clip2_2"], list(seqs[0])) + + seqs = buildSequences(["clip1_1", "clip2_1", "clip1_2", "clip2_2"], numPos=-1) + self.assertEqual(2, len(seqs)) + self.assertEqual(["clip1_1", "clip1_2"], list(seqs[0])) + self.assertEqual(["clip2_1", "clip2_2"], list(seqs[1])) + + seqs = buildSequences([1,3,2], nameFunc=lambda obj: "obj%d"%obj) + self.assertEqual([1,2,3], list(seqs[0].iterObjects())) + self.assertEqual(["obj1", "obj2", "obj3"], list(seqs[0].iterNames())) + + seqs = buildSequences(["dir1/spam1", "dir2/spam1"]) + self.assertEqual(1, len(seqs)) + self.assertEqual(["dir1/spam1", "dir2/spam1"], list(seqs[0])) + + seqs = buildSequences(["dir1/spam1", "dir2/spam1"], assumeFiles=True) + self.assertEqual(2, len(seqs)) + self.assertEqual(["dir1/spam1"], list(seqs[0])) + self.assertEqual(["dir2/spam1"], list(seqs[1])) + +class TestSeqTemplate(unittest.TestCase): + """Test the SeqTemplate class. + """ + + def testSeqTemplate(self): + + t = SeqTemplate("foo#.tif") + self.assertEqual("foo0007.tif", t([7])) + self.assertEqual("foo0008.tif", t.substitute([8])) + self.assertEqual([0], t.expressionIndices(2)) + + t = SeqTemplate("foo#_@...@@.tif") + self.assertEqual("foo0001_2_03.tif", t([1,2,3])) + self.assertEqual([0,1,2], t.expressionIndices(3)) + + t = SeqTemplate("foo#[3]_@[-1]_@@.tif") + self.assertEqual("foo0003_3_03.tif", t([1,2,3])) + self.assertEqual([2,2,2], t.expressionIndices(3)) + + t = SeqTemplate("foo{2*#[3]+2}.tif") + self.assertEqual("foo0008.tif", t([1,2,3])) + self.assertEqual([2], t.expressionIndices(3)) + + +class TestOutputNameGenerator(unittest.TestCase): + """Test the OutputNameGenerator class. + """ + + def testGenerator(self): + seqs = buildSequences(["spam1_1.tif", "spam1_2.tif", "spam1_5.tif"]) + ong = OutputNameGenerator(seqs, "foo") + self.assertEqual([("spam1_1.tif", "foo0001.tif"), + ("spam1_2.tif", "foo0002.tif"), + ("spam1_5.tif", "foo0005.tif")], list(ong)) + + seqs = buildSequences(["spam1_1.tif", "spam1_2.tif", "spam1_5.tif"]) + ong = OutputNameGenerator(seqs, "foo@@_#.tif") + self.assertEqual([("spam1_1.tif", "foo01_0001.tif"), + ("spam1_2.tif", "foo01_0002.tif"), + ("spam1_5.tif", "foo01_0005.tif")], list(ong)) + + seqs = buildSequences(["spam1_1.tif", "spam1_2.tif", "spam1_5.tif"]) + ong = OutputNameGenerator(seqs, "foo_#[2]_{@[1]+2}.tif") + self.assertEqual([("spam1_1.tif", "foo_0001_3.tif"), + ("spam1_2.tif", "foo_0002_3.tif"), + ("spam1_5.tif", "foo_0005_3.tif")], list(ong)) + + seqs = buildSequences(["spam1_1.tif", "spam1_2.tif", "spam1_5.tif"]) + ong = OutputNameGenerator(seqs, "foo@@@.tif") + self.assertEqual([("spam1_1.tif", "foo001.tif"), + ("spam1_2.tif", "foo002.tif"), + ("spam1_5.tif", "foo005.tif")], list(ong)) + + seqs = buildSequences(["spam1_1.tif", "spam1_2.tif", "spam2_5.tif", "spam2_6.tif"]) + ong = OutputNameGenerator(seqs, "foo@...@.tif") + self.assertEqual([("spam1_1.tif", "foo1_1.tif"), + ("spam1_2.tif", "foo1_2.tif"), + ("spam2_5.tif", "foo2_5.tif"), + ("spam2_6.tif", "foo2_6.tif")], list(ong)) + + seqs = buildSequences(["spam2_1.tif", "spam3_1.tif", "spam4_1.tif", "spam5_1.tif"]) + ong = OutputNameGenerator(seqs, "foo@...@.tif", dstRange=Range("5-")) + self.assertEqual([("spam2_1.tif", "foo5_1.tif"), + ("spam3_1.tif", "foo6_1.tif"), + ("spam4_1.tif", "foo7_1.tif"), + ("spam5_1.tif", "foo8_1.tif")], list(ong)) + + seqs = buildSequences(["spam1_1.tif", "spam1_2.tif", "spam2_5.tif", "spam2_6.tif"]) + ong = OutputNameGenerator(seqs, "foo#", dstRange=Range("2-")) + self.assertEqual([("spam1_1.tif", "foo0002.tif"), + ("spam1_2.tif", "foo0003.tif"), + ("spam2_5.tif", "foo0004.tif"), + ("spam2_6.tif", "foo0005.tif")], list(ong)) + + def testSrcRanges(self): + """Test the srcRanges argument. + """ + seqs = buildSequences(["spam1_1.tif", "spam1_2.tif", "spam1_5.tif"]) + ong = OutputNameGenerator(seqs, "foo", srcRanges=[Range("2,5")]) + self.assertEqual([("spam1_2.tif", "foo0002.tif"), + ("spam1_5.tif", "foo0005.tif")], list(ong)) + + def testKeepExt(self): + """Test the keepExt flag. + """ + seqs = buildSequences(["spam1_1.tif", "spam1_2.tif", "spam1_5.tif"]) + ong = OutputNameGenerator(seqs, "foo#.png", keepExt=False) + self.assertEqual([("spam1_1.tif", "foo0001.png"), + ("spam1_2.tif", "foo0002.png"), + ("spam1_5.tif", "foo0005.png")], list(ong)) + + seqs = buildSequences(["spam1_1.tif", "spam1_2.tif", "spam1_5.tif"]) + ong = OutputNameGenerator(seqs, "foo#.png", keepExt=True) + self.assertEqual([("spam1_1.tif", "foo0001.png.tif"), + ("spam1_2.tif", "foo0002.png.tif"), + ("spam1_5.tif", "foo0005.png.tif")], list(ong)) + + def testEnforceDstRange(self): + """Test the enforceDstRange and repeatSrc flags. + """ + seqs = buildSequences(["spam1_1.tif", "spam1_2.tif", "spam1_5.tif"]) + ong = OutputNameGenerator(seqs, "foo#", dstRange=Range("1-5")) + self.assertEqual([("spam1_1.tif", "foo0001.tif"), + ("spam1_2.tif", "foo0002.tif"), + ("spam1_5.tif", "foo0003.tif")], list(ong)) + + seqs = buildSequences(["spam1_1.tif", "spam1_2.tif", "spam1_5.tif"]) + ong = OutputNameGenerator(seqs, "foo#", dstRange=Range("1-7"), enforceDstRange=True) + self.assertEqual([("spam1_1.tif", "foo0001.tif"), + ("spam1_2.tif", "foo0002.tif"), + ("spam1_5.tif", "foo0003.tif"), + ("spam1_1.tif", "foo0004.tif"), + ("spam1_2.tif", "foo0005.tif"), + ("spam1_5.tif", "foo0006.tif"), + ("spam1_1.tif", "foo0007.tif")], list(ong)) + + seqs = buildSequences(["spam1_1.tif", "spam1_2.tif", "spam1_5.tif"]) + ong = OutputNameGenerator(seqs, "foo#", dstRange=Range("1-6"), enforceDstRange=True, repeatSrc=False) + self.assertEqual([("spam1_1.tif", "foo0001.tif"), + ("spam1_2.tif", "foo0002.tif"), + ("spam1_5.tif", "foo0003.tif"), + ("spam1_5.tif", "foo0004.tif"), + ("spam1_5.tif", "foo0005.tif"), + ("spam1_5.tif", "foo0006.tif")], list(ong)) + + +class TestSeqUtils(unittest.TestCase): + """Test the sequence utilities. + """ + + def setUp(self): + for fileName in globmod.glob("tmp/*.txt"): + os.remove(fileName) + for fileName in globmod.glob("tmp/*.ext"): + os.remove(fileName) + + def testSeqCp1(self): + """Test basic seqcp""" + self.createSequence("tmp/spam#.txt", Range("1-5")) + os.system("../utilities/seqcp.py tmp/spam tmp/foo") + self.assertFiles("tmp/foo#.txt", "spam#.txt", Range("1-5")) + self.assertFiles("tmp/spam#.txt", "spam#.txt", Range("1-5")) + + def testSeqCp2(self): + """Test seqcp -s""" + self.createSequence("tmp/spam#.txt", Range("1-5")) + os.system("../utilities/seqcp.py tmp/spam tmp/foo -s2-3") + self.assertFiles("tmp/foo#.txt", "spam#.txt", Range("2-3")) + if os.path.exists("tmp/foo0001.txt"): + self.fail("foo0001.txt exists!") + if os.path.exists("tmp/foo0004.txt"): + self.fail("foo0004.txt exists!") + self.assertFiles("tmp/spam#.txt", "spam#.txt", Range("1-5")) + + def testSeqCp3(self): + """Test seqcp of discontinuous sequence""" + self.createSequence("tmp/spam#.txt", Range("1-2,5")) + os.system("../utilities/seqcp.py tmp/spam tmp/foo") + self.assertFiles("tmp/foo#.txt", "spam#.txt", Range("1,2,5")) + self.assertFiles("tmp/spam#.txt", "spam#.txt", Range("1,2,5")) + + def testSeqCp4(self): + """Test seqcp -d""" + self.createSequence("tmp/spam#.txt", Range("1-2,5")) + os.system("../utilities/seqcp.py tmp/spam tmp/foo -d2-") + self.assertFiles("tmp/foo#.txt", "spam#.txt", Range("2-4"), Range("1,2,5")) + self.assertFiles("tmp/spam#.txt", "spam#.txt", Range("1,2,5")) + + + def testSeqMv1(self): + """Test basic seqmv""" + self.createSequence("tmp/spam#.txt", Range("1-5")) + os.system("../utilities/seqmv.py tmp/spam tmp/foo") + self.assertFiles("tmp/foo#.txt", "spam#.txt", Range("1-5")) + + def testSeqMv2(self): + """Test seqmv -s""" + self.createSequence("tmp/spam#.txt", Range("1-5")) + os.system("../utilities/seqmv.py tmp/spam tmp/foo -s2-3") + self.assertFiles("tmp/foo#.txt", "spam#.txt", Range("2-3")) + if os.path.exists("tmp/foo0001.txt"): + self.fail("foo0001.txt exists!") + if os.path.exists("tmp/foo0004.txt"): + self.fail("foo0004.txt exists!") + + def testSeqMv3(self): + """Test seqmv of discontinuous sequence""" + self.createSequence("tmp/spam#.txt", Range("1-2,5")) + os.system("../utilities/seqmv.py tmp/spam tmp/foo") + self.assertFiles("tmp/foo#.txt", "spam#.txt", Range("1,2,5")) + + def testSeqMv4(self): + """Test seqmv -d""" + self.createSequence("tmp/spam#.txt", Range("1-2,5")) + os.system("../utilities/seqmv.py tmp/spam tmp/foo -d2-") + self.assertFiles("tmp/foo#.txt", "spam#.txt", Range("2-4"), Range("1,2,5")) + + def testSeqMv5(self): + """Test seqmv of overlapping src/dst ranges (move backward)""" + self.createSequence("tmp/spam#.txt", Range("2-8")) + os.system("../utilities/seqmv.py tmp/spam tmp/spam -d1-") + self.assertFiles("tmp/spam#.txt", "spam#.txt", Range("1-7"), Range("2-8")) + + def testSeqMv6(self): + """Test seqmv of overlapping src/dst ranges (move forward)""" + self.createSequence("tmp/spam#.txt", Range("2-8")) + os.system("../utilities/seqmv.py tmp/spam tmp/spam -d3-") + self.assertFiles("tmp/spam#.txt", "spam#.txt", Range("3-9"), Range("2-8")) + + def testSeqMv7(self): + """Test seqmv of overlapping src/dst ranges (needs tmp sequence)""" + self.createSequence("tmp/spam#.txt", Range("2-10x2")) + os.system("../utilities/seqmv.py tmp/spam tmp/spam -d4-") + self.assertFiles("tmp/spam#.txt", "spam#.txt", Range("4-8"), Range("2-10x2")) + + def testSeqMv8(self): + """Test the -e option (to change the extension).""" + self.createSequence("tmp/spam#.ext", Range("1-3")) + os.system('../utilities/seqmv.py "tmp/spam*.ext" tmp/foo@@.txt -e') + self.assertFiles("tmp/foo@@.txt", "spam#.ext", Range("1-3")) + + + def assertFiles(self, namePattern, origPattern, rng, origRng=None): + if origRng is None: + origRng = rng + + srcTmpl = SeqTemplate(namePattern) + origTmpl = SeqTemplate(origPattern) + for i,j in zip(rng, origRng): + self.assertFile(srcTmpl([i]), origTmpl([j])) + + def assertFile(self, name, origName): + """Assert that a file exists and originated from a particular file. + + name: File name to check. + origName: The (base) name of the original file. + """ + if not os.path.exists(name): + self.fail("File %s does not exist"%name) + f = open(name, "rt") + name = f.read() + f.close() + self.assertEqual(origName, name) + + def createSequence(self, pattern, rng): + """Create a file sequence. + + pattern is a string containing the name pattern (must contain one + number substitution pattern). rng is a Range object containing the + range of frames that should be generated. + """ + tmpl = SeqTemplate(pattern) + for i in rng: + name = tmpl([i]) + f = open(name, "wt") + f.write(os.path.basename(name)) + f.close() + + +###################################################################### + +if __name__=="__main__": + unittest.main() Modified: cgkit/trunk/utilities/seqcp.py =================================================================== --- cgkit/trunk/utilities/seqcp.py 2009-08-06 20:01:27 UTC (rev 329) +++ cgkit/trunk/utilities/seqcp.py 2009-08-08 09:09:38 UTC (rev 330) @@ -36,6 +36,7 @@ import sys import optparse +import cgkit.cgkitinfo from cgkit import sequence @@ -61,11 +62,19 @@ parser = optparse.OptionParser(usage="%prog [options] src dst") parser.add_option("-s", "--source-frames", default="0-", metavar="FRAMES", help="Specify a subset of the source frames") parser.add_option("-d", "--destination-frames", default=None, metavar="FRAMES", help="Specify the destination numbers") + parser.add_option("-e", "--drop-extensions", action="store_true", default=False, help="Don't handle missing file extensions in the output pattern") + parser.add_option("-S", "--symlink", action="store_true", default=False, help="Create symbolic links instead of copying the files") + parser.add_option("-R", "--realpath", action="store_true", default=False, help="Use the real path of the source files (follow links). This is only relevant when symbolic links are created.") parser.add_option("-f", "--force", action="store_true", default=False, help="Never query the user for confirmation") parser.add_option("-t", "--test", action="store_true", default=False, help="Only print what would be done, but don't copy anything") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Print every file when it is copied") + parser.add_option("-V", "--version", action="store_true", default=False, help="Display version information") opts,args = parser.parse_args() + if opts.version: + print ("seqcp (cgkit %s)"%cgkit.cgkitinfo.version) + sys.exit(0) + if len(args)!=2: parser.print_usage() return @@ -84,19 +93,26 @@ # Determine the source sequences fseqs = sequence.glob(srcSeq) - copier = sequence.CopySequence(fseqs, dstArg, [srcRange], dstRange, verbose=opts.verbose) + if opts.symlink: + processor = sequence.SymLinkSequence(fseqs, dstArg, [srcRange], dstRange, + keepExt=not opts.drop_extensions, verbose=opts.verbose, resolveSrcLinks=opts.realpath) + opStr = "Link" + else: + processor = sequence.CopySequence(fseqs, dstArg, [srcRange], dstRange, + keepExt=not opts.drop_extensions, verbose=opts.verbose, resolveSrcLinks=opts.realpath) + opStr = "Copy" - for srcSeq,dstSeq in copier.sequences(): - print ("Copy: %s -> %s"%(srcSeq, dstSeq)) + for srcSeq,dstSeq in processor.sequences(): + print ("%s: %s -> %s"%(opStr, srcSeq, dstSeq)) # Check a file number would get appended to a trailing number in the base name - if copier.mergesNumbers() and not opts.force: + if processor.mergesNumbers() and not opts.force: print ("WARNING: The destination name ends in a number which would affect the output sequence number.") if not opts.test and not promptUser("Are you sure to continue?"): return # Check if an existing file would get overwritten - overwrites = list(copier.overwrites()) + overwrites = list(processor.overwrites()) if len(overwrites)>0 and not opts.force: print ("WARNING: %s files would get overwritten."%len(overwrites)) if not opts.test and not promptUser("Are you sure to continue?"): @@ -104,9 +120,9 @@ # Dry run or real run... if opts.test: - copier.dryRun() + processor.dryRun() else: - copier.run() + processor.run() # TODO: If forward/backward copy fails, the files need to be copied to a temporary sequence first and then renamed. Modified: cgkit/trunk/utilities/seqls.py =================================================================== --- cgkit/trunk/utilities/seqls.py 2009-08-06 20:01:27 UTC (rev 329) +++ cgkit/trunk/utilities/seqls.py 2009-08-08 09:09:38 UTC (rev 330) @@ -38,6 +38,7 @@ import sys, os, glob import os.path import time +import cgkit.cgkitinfo from cgkit import sequence class SequenceInfo: @@ -130,8 +131,13 @@ parser = optparse.OptionParser(usage="%prog [options] paths") parser.add_option("-l", "--long", action="store_true", default=False, help="Print additional information per sequence") parser.add_option("-d", "--directories", action="store_true", default=False, help="List directories") + parser.add_option("-V", "--version", action="store_true", default=False, help="Display version information") opts,args = parser.parse_args() + if opts.version: + print ("seqls (cgkit %s)"%cgkit.cgkitinfo.version) + sys.exit(0) + if len(args)==0: args = ["*"] Modified: cgkit/trunk/utilities/seqmv.py =================================================================== --- cgkit/trunk/utilities/seqmv.py 2009-08-06 20:01:27 UTC (rev 329) +++ cgkit/trunk/utilities/seqmv.py 2009-08-08 09:09:38 UTC (rev 330) @@ -36,6 +36,7 @@ import sys import optparse +import cgkit.cgkitinfo from cgkit import sequence def promptUser(question): @@ -60,11 +61,17 @@ parser = optparse.OptionParser(usage="%prog [options] src dst") parser.add_option("-s", "--source-frames", default="0-", metavar="FRAMES", help="Specify a subset of the source frames") parser.add_option("-d", "--destination-frames", default=None, metavar="FRAMES", help="Specify the destination numbers") + parser.add_option("-e", "--drop-extensions", action="store_true", default=False, help="Don't handle missing file extensions in the output pattern") parser.add_option("-f", "--force", action="store_true", default=False, help="Never query the user for confirmation") parser.add_option("-t", "--test", action="store_true", default=False, help="Only print what would be done, but don't move anything") - parser.add_option("-v", "--verbose", action="store_true", default=False, help="Print every file when it is copied") + parser.add_option("-v", "--verbose", action="store_true", default=False, help="Print every file when it is moved") + parser.add_option("-V", "--version", action="store_true", default=False, help="Display version information") opts,args = parser.parse_args() + if opts.version: + print ("seqmv (cgkit %s)"%cgkit.cgkitinfo.version) + sys.exit(0) + if len(args)!=2: parser.print_usage() return @@ -83,7 +90,7 @@ # Determine the source sequences fseqs = sequence.glob(srcSeq) - mover = sequence.MoveSequence(fseqs, dstArg, [srcRange], dstRange, verbose=opts.verbose) + mover = sequence.MoveSequence(fseqs, dstArg, [srcRange], dstRange, keepExt=not opts.drop_extensions, verbose=opts.verbose) for srcSeq,dstSeq in mover.sequences(): print ("Move: %s -> %s"%(srcSeq, dstSeq)) Modified: cgkit/trunk/utilities/seqrm.py =================================================================== --- cgkit/trunk/utilities/seqrm.py 2009-08-06 20:01:27 UTC (rev 329) +++ cgkit/trunk/utilities/seqrm.py 2009-08-08 09:09:38 UTC (rev 330) @@ -38,6 +38,7 @@ import optparse import os import os.path +import cgkit.cgkitinfo from cgkit import sequence def promptUser(question): @@ -59,8 +60,13 @@ parser = optparse.OptionParser(usage="%prog [options] sequences") parser.add_option("-f", "--force", action="store_true", default=False, help="Never query the user for confirmation") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Print every file when it is deleted") + parser.add_option("-V", "--version", action="store_true", default=False, help="Display version information") opts,args = parser.parse_args() + if opts.version: + print ("seqrm (cgkit %s)"%cgkit.cgkitinfo.version) + sys.exit(0) + if len(args)<1: parser.print_usage() return This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. 
 [cgkit-commits] SF.net SVN: cgkit:[337] cgkit/trunk From: - 2009-08-20 20:57:10 Revision: 337 http://cgkit.svn.sourceforge.net/cgkit/?rev=337&view=rev Author: mbaas Date: 2009-08-20 20:56:55 +0000 (Thu, 20 Aug 2009) Log Message: ----------- Added the (preliminary) jobqueue module. Modified Paths: -------------- cgkit/trunk/setup.py Added Paths: ----------- cgkit/trunk/cgkit/jobqueue/ cgkit/trunk/cgkit/jobqueue/__init__.py cgkit/trunk/cgkit/jobqueue/defaultprocs/ cgkit/trunk/cgkit/jobqueue/defaultprocs/__init__.py cgkit/trunk/cgkit/jobqueue/defaultprocs/blender.py cgkit/trunk/cgkit/jobqueue/defaultprocs/dumpenv.py cgkit/trunk/cgkit/jobqueue/defaultprocs/maya.py cgkit/trunk/cgkit/jobqueue/defaultprocs/renderrib.py cgkit/trunk/cgkit/jobqueue/jobhandle.py cgkit/trunk/cgkit/jobqueue/jobproc.py cgkit/trunk/cgkit/jobqueue/jobqueue.py cgkit/trunk/unittests/test_jobqueue.py Added: cgkit/trunk/cgkit/jobqueue/__init__.py =================================================================== --- cgkit/trunk/cgkit/jobqueue/__init__.py (rev 0) +++ cgkit/trunk/cgkit/jobqueue/__init__.py 2009-08-20 20:56:55 UTC (rev 337) @@ -0,0 +1,46 @@ +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is the Python Computer Graphics Kit. +# +# The Initial Developer of the Original Code is Matthias Baas. +# Portions created by the Initial Developer are Copyright (C) 2009 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +import os +from jobqueue import createJobQueue, JobQueue, JobQueueError +from jobproc import JobProc + +def defaultJobQueueLocation(): + """Return the directory location of the default job queue. + + The default queue can be set via the CGKIT_JOB_QUEUE environment variable. + Returns None if there is no default location. + """ + return os.getenv("CGKIT_JOB_QUEUE") Added: cgkit/trunk/cgkit/jobqueue/defaultprocs/blender.py =================================================================== --- cgkit/trunk/cgkit/jobqueue/defaultprocs/blender.py (rev 0) +++ cgkit/trunk/cgkit/jobqueue/defaultprocs/blender.py 2009-08-20 20:56:55 UTC (rev 337) @@ -0,0 +1,155 @@ +# Render using Blender + +import os, os.path, subprocess, sys, traceback +import cgkit.jobqueue + +class blender(cgkit.jobqueue.JobProc): + """Render a Blender file. + """ + + def __init__(self, blendFile=None, output=None, frames=None, format=None, + addExtension=None, sceneName=None, threads=None, + blenderCmd = "blender", + label=None): + """Constructor. + + blendFile: The file to render (option -b) + output: The output path and name (option -o). + frames: This is either a single int to render a single frame, a tuple + (start,end) to render a frame range or None to render everything. + The start or the end may be None to indicate the start or end + of the entire animation. + format: A string describing the output format (option -F). + addExtension: A boolean indicating whether a file extension should be added + or not (option -x). + sceneName: The scene name (option -S) + threads: The number of threads that should be used for rendering (option -t). + 0=system processor count. + blenderCmd: The name of the Blender executable + label: The GUI label string + """ + # Check the parameter types... + if not isinstance(blendFile, basestring): + raise TypeError("blendFile argument must be a string, got a %s instead"%getattr(type(blendFile), "__name__", "?")) + if not (output is None or isinstance(output, basestring)): + raise TypeError("output argument must be a string or None, got a %s instead"%getattr(type(output), "__name__", "?")) + if not (format is None or isinstance(format, basestring)): + raise TypeError("format argument must be a string or None, got a %s instead"%getattr(type(format), "__name__", "?")) + if not (addExtension is None or type(addExtension) is bool): + raise TypeError("format argument must be a string or None, got a %s instead"%getattr(type(addExtension), "__name__", "?")) + if not (sceneName is None or isinstance(sceneName, basestring)): + raise TypeError("sceneName argument must be a string or None, got a %s instead"%getattr(type(sceneName), "__name__", "?")) + if not (threads is None or type(threads) is int): + raise TypeError("threads argument must be an int or None, got a %s instead"%getattr(type(threads), "__name__", "?")) + if not isinstance(blenderCmd, basestring): + raise TypeError("blenderCmd argument must be a string, got a %s instead"%getattr(type(blenderCmd), "__name__", "?")) + if frames is None: + startFrame = None + endFrame = None + elif type(frames) is int: + startFrame = frames + endFrame = frames + else: + try: + s,e = frames + except: + raise TypeError("frames argument must be an int, two ints or None") + if not (s is None or type(s) is int): + raise TypeError("The start frame must be an int or None, got a %s instead"%getattr(type(s), "__name__", "?")) + if not (e is None or type(e) is int): + raise TypeError("The end frame must be an int or None, got a %s instead"%getattr(type(e), "__name__", "?")) + startFrame = s + endFrame = e + + # Set the default label if no label was given + if label is None: + label = 'Blender "%s"'%os.path.basename(blendFile) + + cgkit.jobqueue.JobProc.__init__(self, label=label) + + # Store the parameters + self._blenderExec = blenderCmd + self._blendFile = blendFile + self._output = output + self._startFrame = startFrame + self._endFrame = endFrame + self._format = format + self._addExtension = addExtension + self._sceneName = sceneName + self._threads = threads + + self._currentFrame = None + + self._hasError = False + self._errorLine = "" + + def run(self): + """Run Blender. + """ + cmd = self.createCommandLine() + print >>self.stdout, cmd + ret = self.execCmd(cmd) + if ret!=0: + print >>self.stderr, "\nReturn code:",ret + self._hasError = True + self._errorLine = "Return code: %s"%ret + + if self._hasError: + self.setError() + self.setStatusLine(self._errorLine) + else: + self.setStatusLine("Done") + + + def stdoutCallback(self, lineNr, line): + """Scan stdout for errors. + """ + self.setStatusLine(line.strip()) + if line.startswith("ERROR:") and not self._hasError: + print >>self.stderr, "stdout, line %s:\n%s"%(lineNr+1, line) + print >>self.stderr, "(see stdout for more details)" + self._hasError = True + self._errorLine = line + elif line.startswith("Fra:"): + try: + frame = int(line[4:].split()[0]) + except: + frame = self._currentFrame + + if frame!=self._currentFrame: + self._currentFrame = frame + if self._startFrame is not None and self._endFrame is not None: + pc = 100*(frame-self._startFrame)/(self._endFrame-self._startFrame+1) + self.setProgress(pc) + + def stderrCallback(self, lineNr, line): + """Scan stderr for errors. + """ + if not self._hasError and line.strip()!="": + self._hasError = True + + def createCommandLine(self): + """Return the command line for invoking Blender. + """ + toks = [self._blenderExec] + toks.append('-b "%s"'%self._blendFile) + if self._sceneName is not None: + toks.append('-S "%s"'%self._sceneName) + if self._output is not None: + toks.append('-o "%s"'%self._output) + if self._addExtension is not None: + toks.append("-x %d"%int(self._addExtension)) + if self._format is not None: + toks.append("-F %s"%self._format) + if self._threads is not None: + toks.append("-t %s"%self._threads) + if self._startFrame is not None and self._startFrame==self._endFrame: + toks.append("-f %d"%self._startFrame) + else: + if self._startFrame is not None: + toks.append("-s %s"%self._startFrame) + if self._endFrame is not None: + toks.append("-e %s"%self._endFrame) + toks.append("-a") + + return " ".join(toks) Added: cgkit/trunk/cgkit/jobqueue/defaultprocs/dumpenv.py =================================================================== --- cgkit/trunk/cgkit/jobqueue/defaultprocs/dumpenv.py (rev 0) +++ cgkit/trunk/cgkit/jobqueue/defaultprocs/dumpenv.py 2009-08-20 20:56:55 UTC (rev 337) @@ -0,0 +1,20 @@ +# Test job to dump the environment to stdout + +import os, socket, getpass +import cgkit.jobqueue + +class dumpenv(cgkit.jobqueue.JobProc): + """Render a Blender file. + """ + + def __init__(self): + cgkit.jobqueue.JobProc.__init__(self, label="Dump Environment") + + def run(self): + print ("Host name : %s"%socket.gethostname()) + print ("User name : %s"%getpass.getuser()) + print ("Current dir : %s"%os.getcwd()) + print ("\nEnvironment variables:\n") + vars = os.environ.keys() + for var in sorted(vars): + print (" %s = %s"%(var, os.environ.get(var))) Added: cgkit/trunk/cgkit/jobqueue/defaultprocs/maya.py =================================================================== --- cgkit/trunk/cgkit/jobqueue/defaultprocs/maya.py (rev 0) +++ cgkit/trunk/cgkit/jobqueue/defaultprocs/maya.py 2009-08-20 20:56:55 UTC (rev 337) @@ -0,0 +1,75 @@ +# Render using Maya + +import os, os.path, subprocess, sys, traceback +import cgkit.jobqueue + +class maya(cgkit.jobqueue.JobProc): + """Render a Maya file. + """ + + def __init__(self, sceneFile=None, project=None, + startFrame=None, endFrame=None, + renderCmd = "Render", + label=None): + """Constructor. + + label: The GUI label string + """ + # Check the parameter types... + if not isinstance(sceneFile, basestring): + raise TypeError("sceneFile argument must be a string, got a %s instead"%getattr(type(sceneFile), "__name__", "?")) + if not (project is None or isinstance(project, basestring)): + raise TypeError("project argument must be a string or None, got a %s instead"%getattr(type(project), "__name__", "?")) + if not (startFrame is None or type(startFrame) in [int, float]): + raise TypeError("startFrame must be a number type or None, got a %s instead"%getattr(type(startFrame), "__name__", "?")) + if not (endFrame is None or type(endFrame) in [int, float]): + raise TypeError("endFrame must be a number type or None, got a %s instead"%getattr(type(endFrame), "__name__", "?")) + if not isinstance(renderCmd, basestring): + raise TypeError("renderCmd argument must be a string, got a %s instead"%getattr(type(renderCmd), "__name__", "?")) + + # Set the default label if no label was given + if label is None: + label = 'Maya "%s"'%os.path.basename(sceneFile) + + cgkit.jobqueue.JobProc.__init__(self, label=label) + + # Store the parameters + toks = [renderCmd] + if startFrame is not None: + toks.append("-s %s"%startFrame) + if endFrame is not None: + toks.append("-e %s"%endFrame) + if project is not None and project!="": + toks.append('-proj "%s"'%project) + toks.append('"%s"'%sceneFile) + + self._cmdLine = " ".join(toks) + + def run(self): + """Run the Render tool. + """ + print >>self.stdout, self._cmdLine + ret = self.execCmd(self._cmdLine) + if ret!=0: + print >>self.stderr, "\nReturn code:",ret + self.setError() + else: + self.setStatusLine("Done") + + def stdoutCallback(self, lineNr, line): + """Scan stdout for status/errors. + """ + pass + + def stderrCallback(self, lineNr, line): + """Scan stderr for status/errors. + """ + # Does a frame begin? + if line.startswith("Starting Rendering"): + # Get the image path (remove the prefix and the trailing dot) + fileName = os.path.normpath(line.strip()[19:-1]) + p,n = os.path.split(fileName) + self.setStatusLine("Rendering %s (%s)"%(n,p)) + # Is there an error line? + elif line.startswith("Error:"): + self.setStatusLine(line.strip()) Added: cgkit/trunk/cgkit/jobqueue/defaultprocs/renderrib.py =================================================================== --- cgkit/trunk/cgkit/jobqueue/defaultprocs/renderrib.py (rev 0) +++ cgkit/trunk/cgkit/jobqueue/defaultprocs/renderrib.py 2009-08-20 20:56:55 UTC (rev 337) @@ -0,0 +1,44 @@ +# Render a RIB file + +import os, subprocess +import cgkit.jobqueue + +class renderrib(cgkit.jobqueue.JobProc): + + def __init__(self, rib, renderer="aqsis"): + cgkit.jobqueue.JobProc.__init__(self) + self._rib = rib + self._renderer = renderer.lower() + + def run(self): + args = [self._rendererExecutable(), self._rib] + cmd = " ".join(args) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=True) + outdata,errdata = proc.communicate() + outdata = outdata.rstrip() + errdata = errdata.rstrip() + print "OUT" + print outdata + print "ERR" + print errdata + print "RET" + print proc.returncode + + if proc.returncode!=0 or errdata!="": + self.setError() + + + def _rendererExecutable(self): + """Return the name of the renderer executable. + """ + if self._renderer=="aqsis": + return "aqsis" + elif self._renderer=="pixie": + return "rndr" + elif self._renderer=="3delight": + return "renderdl" + elif self._renderer=="prman": + return "prman" + else: + raise ValueError("Unknown renderer: %s"%self._renderer) Added: cgkit/trunk/cgkit/jobqueue/jobhandle.py =================================================================== --- cgkit/trunk/cgkit/jobqueue/jobhandle.py (rev 0) +++ cgkit/trunk/cgkit/jobqueue/jobhandle.py 2009-08-20 20:56:55 UTC (rev 337) @@ -0,0 +1,367 @@ +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is the Python Computer Graphics Kit. +# +# The Initial Developer of the Original Code is Matthias Baas. +# Portions created by the Initial Developer are Copyright (C) 2009 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +import os, os.path, glob + + +class JobHandle(object): + """Job object (queued). + + Objects of this class represent jobs that have already been queued. + + Every job has: + + - A type name + - An instance name + - A job description + - An associated directory + """ + + def __init__(self, location, rootLocation): + """Constructor. + + location is the path to the job directory (this may also point to a + link file). + rootLocation is the path to the job root directory (this is used + for resolving job relative relative links). + """ + object.__init__(self) + # The job location (if useSymLinks is False, this may also point to a link file) + self._location = str(location) + + # The job root directory + self._rootLocation = rootLocation + + # The resolved location. This is always a directory (or technically, it + # could be a non-existent path). If useSymLinks is True, this is + # always the same as the above location. + self._realLocation = self._resolveTxtLinks(self._location) + + + def _resolveTxtLinks(self, location): + """Resolve a path so that it points to a directory and not to a link file. + + If location already points to a directory, it is returned unmodified. + If it points to a link file, the file is followed and the process + repeats until a directory is encountered or the targegt does not exist. + """ + visited = {} + while os.path.exists(location) and os.path.isfile(location): + visited[location] = True + location = self._getTxtLinkTarget(location) + if location in visited: + raise ValueError("Cyclic directory links detected") + return location + + def _getTxtLinkTarget(self, linkFileName): + """Return the target that a link file points to. + """ + f = open(linkFileName, "rt") + # Read the first line which must be "[link]" + s = f.readline() + if s!="[link]\n": + raise ValueError("%s is not a link file"%linkFileName) + # Read the line containing the target + targetLine = f.readline() + f.close() + # Check the target line + if not targetLine.startswith("target="): + raise ValueError("Error in link file %s"%linkFileName) + target = targetLine[7:].strip() + # The link must always be relative to the job root + if not target.startswith("$JOBROOT"): + raise ValueError("Invalid link target: %s"%target) + target = self._rootLocation+target[8:] + return target + + def __str__(self): + if self.isFinished(): + state = "finished" + elif self.isRunning(): + state = "running" + elif self.isWaiting(): + state = "waiting" + else: + state = "?" + + res = "%s (%s)"%(os.path.basename(self._location), state) + return res + + @property + def location(self): + """Return the job directory. + """ + return self._location + + @property + def label(self): + """Return a short job label to quickly identify the job. + """ + try: + try: + f = open(self.labelFile, "rt") + label = f.read().strip() + finally: + f.close() + except: + label = "" + + if label=="": + label = "Job %s"%self.number + + return label + + @property + def number(self): + """Return the job number. + """ + nrStr = os.path.basename(self._location)[3:] + try: + return int(nrStr) + except: + return -1 + + @property + def submitTime(self): + """Return the submission time (in seconds). + + May return None if the job doesn't exist or is broken. + """ + try: + s = os.stat(self.procDefFile) + return s.st_mtime + except OSError: + return None + + @property + def startTime(self): + """Return the time the job was started (in seconds). + + Returns None if the job hasn't been started yet (or there is a problem + with the job directory). + """ + try: + s = os.stat(self.pidFile) + return s.st_mtime + except OSError: + return None + + @property + def endTime(self): + """Return the time the job was finished (in seconds). + + Returns None if the job hasn't been finished yet (or there is a problem + with the job directory). + """ + try: + s = os.stat(self.finishedDir) + return s.st_mtime + except OSError: + return None + + @property + def progress(self): + """Return the progress percentage value as an int. + + Returns None if the value couldn't be obtained for some reason. + """ + try: + progressFile = self.progressFile + if os.path.exists(progressFile): + res = os.path.getsize(self.progressFile) + if res>100: + res = 100 + else: + res = 0 + return res + except: + return None + + @property + def statusLine(self): + """Return a string containing the current status line for the GUI. + + The status line indicates what a running job is currently doing. + """ + try: + try: + f = open(self.statusLineFile, "rt") + line = f.read().strip() + finally: + f.close() + except: + line = "" + + return line + + + @property + def finishedDir(self): + return os.path.join(self._realLocation, ".finished") + + @property + def runningDir(self): + return os.path.join(self._realLocation, ".running") + + @property + def procDefFile(self): + return os.path.join(self._realLocation, ".proc_def") + + @property + def labelFile(self): + """Return the location of the file that contains the job label. + """ + return os.path.join(self._realLocation, ".label") + + @property + def pidFile(self): + """Return the location of the PID file. + + This file contains the PID of the process that is/was running the job. + """ + return os.path.join(self.runningDir, ".pid") + + @property + def hostFile(self): + """Return the location of the host file. + + The host file contains the name of the host that is/was running the job. + """ + return os.path.join(self.runningDir, ".host") + + @property + def progressFile(self): + """Return the location of the progress indicator file. + """ + return os.path.join(self.runningDir, ".progress") + + @property + def statusLineFile(self): + """Return the location of the status line file. + """ + return os.path.join(self.runningDir, ".statusline") + + @property + def errorMarkerFile(self): + """Return the location of the error marker file. + + The presence of this file indicates that running the job resulted + in an error. + """ + return os.path.join(self.runningDir, ".error_marker") + + @property + def procTracebackFile(self): + """Return the location of the file that contains the job procedure traceback. + """ + return os.path.join(self.runningDir, ".proc_traceback") + + @property + def stdoutFile(self): + """Return the location of the file that contains stdout. + """ + return os.path.join(self.runningDir, ".stdout") + + @property + def stderrFile(self): + """Return the location of the file that contains stderr. + """ + return os.path.join(self.runningDir, ".stderr") + + def isWaiting(self): + """Check if this job is currently in 'waiting' state. + """ + if os.path.exists(self.finishedDir): + return False + if os.path.exists(self.runningDir): + return False + return os.path.exists(self.procDefFile) + + def isRunning(self): + """Check if this job is currently in 'running' state. + """ + if self.isFinished(): + return False + else: + return os.path.exists(self.runningDir) + + def isFinished(self): + """Check if this job is currently in 'finished' state. + """ + return os.path.exists(self.finishedDir) + + def listSubJobs(self): + """Return a list of sub-jobs. + + The return value is a list of JobHandle objects. + """ + subDirs = glob.glob(os.path.join(self._realLocation, "job*")) + #subDirs = filter(lambda p: os.path.isdir(p), subDirs) + jobs = map(lambda jobDir: (jobDir, int(os.path.basename(jobDir)[3:])), subDirs) + jobs.sort(key=lambda a: a[1]) + return [JobHandle(jobDir, self._rootLocation) for jobDir,nr in jobs] + + def hasError(self, recursive=False): + """Check if this job produced an error. + + If recursive is False, only this job is considered, not the children + jobs. In this case, the return value is only meaningful when the job + is in finished state. + If recursive is True, the result will be True if any job in the entire + sub-hierarchy has an error. This can be an expensive operation because + all sub-directories have to be checked. + """ + err = os.path.exists(self.errorMarkerFile) + if err: + return True + + if recursive: + subJobs = self.listSubJobs() + for j in subJobs: + if j.hasError(recursive=True): + return True + + return False + + def setError(self): + """Set the error flag. + + This method may only be called by the process that is currently + running the job (the .running directory must exist). + """ + errFile = self.errorMarkerFile + if not os.path.exists(errFile): + f = open(errFile, "wb") + f.close() Added: cgkit/trunk/cgkit/jobqueue/jobproc.py =================================================================== --- cgkit/trunk/cgkit/jobqueue/jobproc.py (rev 0) +++ cgkit/trunk/cgkit/jobqueue/jobproc.py 2009-08-20 20:56:55 UTC (rev 337) @@ -0,0 +1,273 @@ +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is the Python Computer Graphics Kit. +# +# The Initial Developer of the Original Code is Matthias Baas. +# Portions created by the Initial Developer are Copyright (C) 2009 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +import os.path, threading, subprocess + + +class _StreamPipe(threading.Thread): + """Read from a pipe (stdout/stderr) and write the data into a file. + + This is used to read the stdout/stderr streams from a process launched + via the subprocess module. + """ + + def __init__(self, pipe, outputFile, lineCallback=None, errStream=None, startLineNr=1, name=None): + """Constructor. + + pipe is the stdout or stdin pipe of the launched process. + outputFile is an open file object that will receive the data that + is read from pipe. + lineCallback is a callable (or None) that gets called for every + line read from pipe. The callback is called with two arguments, + the line number of the current line and the line itself. + errStream is a file-like object that receives error messages. If None + is passed then sys.stderr is used. + name is the name of the thread. + """ + threading.Thread.__init__(self, name=name) + self.setDaemon(True) + self._pipe = pipe + self._outputFile = outputFile + self._lineCallback = lineCallback + if errStream is None: + errStream = sys.stderr + self._errStream = errStream + self._startLineNr = startLineNr-1 + self.numLines = None + + def run(self): + lines = self._startLineNr + while 1: + s = self._pipe.readline() + if s=="": + self.numLines = lines + return + if self._outputFile is not None: + try: + self._outputFile.write(s) + except: + traceback.print_exc(file=self._errStream) + self._outputFile = None + lines += 1 + # Call the line callback function + if self._lineCallback is not None: + try: + self._lineCallback(lines, s) + except: + traceback.print_exc(file=self._errStream) + self._lineCallback = None + + +class JobProc(object): + """Job procedure base class. + """ + + def __init__(self, label=None): + """Constructor. + + Note that for one particular job, the job procedure may get created + several times (e.g. just to validate parameters). + """ + object.__init__(self) + # As a convenience for the user this is set outside the class + # (in Job._initJobDir()) instead of postCreate()) + self._jobDir = None + self._jobHandle = None + if label is not None: + label = str(label) + self.label = label + self.stdout = None + self.stderr = None + + def run(self): + """Do whatever this job has to do. + + When this method is run, the current directory is set to the + corresponding :file:.running directory. + This method has to be implemented in a derived class. The base + method does nothing. + + The method can use the :meth:execCmd() method to run + command line applications. It can call :meth:setStatusLine() + and :meth:setProgress() to report status + information to the user and it can call :meth:setError() + when the job has failed to run successfully. + """ + pass + + def postCreate(self, jobDir): + """This method is called once when the job is created. + + *jobDir* is the directory in which the job will be run (the attribute + :attr:runningDir will not yet be initialised). + + This method can be implemented in a derived class. The base method + does nothing. + + Note: The method is not executed when the job is actually run, so you + can use it to do initializations that must only be run exactly once. + """ + pass + + def execCmd(self, cmd): + """Execute a command line string. + + *cmds* is a string containing a command that will be executed. + stdout/stderr is captured into the job's stdout/stderr stream. + The return value is the return code that was returned by the command. + + Derived classes can implement :meth:stdoutCallback() + and :meth:stderrCallback() which get called + for every line. + """ + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + stdoutReader = _StreamPipe(proc.stdout, self.stdout, lineCallback=self.stdoutCallback, errStream=self.stderr, name="StdOutReader") + stderrReader = _StreamPipe(proc.stderr, self.stderr, lineCallback=self.stderrCallback, errStream=self.stderr, name="StdErrReader") + stdoutReader.start() + stderrReader.start() + ret = proc.wait() + return ret + + def stdoutCallback(self, lineNr, line): + """This method gets called for every line written to stdout. + + When :meth:execCmd() is used to run a command + line application, then this callback is called for every line written + by the application to stdout. + + *lineNr* is the line number and *line* the actual contents of the line. + + Derived classes may implement this method to scan for errors, status + messages or progress reports. The default implementation does nothing. + """ + pass + + def stderrCallback(self, lineNr, line): + """This method gets called for every line written to stderr. + + When :meth:execCmd() is used to run a command + line application, then this callback is called for every line written + by the application to stderr. + + *lineNr* is the line number and *line* the actual contents of the line. + + Derived classes may implement this method to scan for errors, status + messages or progress reports. The default implementation does nothing. + """ + pass + + def setProgress(self, value): + """Set the progress percentage value. + + *value* is a number between 0 and 100 (it is clamped) which indicates + the progress the job has made. + """ + value = int(value) + # Clamp between 0 and 100 + value = max(0, min(value, 100)) + progressFile = self._jobHandle.progressFile + if os.path.exists(progressFile): + size = os.path.getsize(progressFile) + if size0: + try: + f = open(progressFile, "ab") + except: + print >>sys.stderr, "Failed to update progress file "%progressFile + return + try: + f.write(n*"*") + finally: + f.close() + + def setStatusLine(self, s): + """Set a status line string. + + *s* is a string with a short message that reflects the current status + of the job. + """ + s = str(s) + statusLineFile = self._jobHandle.statusLineFile + try: + f = open(statusLineFile, "wt") + except: + print >>sys.stderr, "Failed to update progress file "%progressFile + return + try: + f.write(s) + finally: + f.close() + + def setError(self): + """Set the job's error flag. + """ + self._jobHandle.setError() + + @property + def runningDir(self): + """The directory in which the job is run. + """ + return self._jobHandle.runningDir + + def _begin(self, jobHandle): + """This is called internally before run() is called. + + jobHandle is the JobHandle object that is associated with this job. + """ + self._jobHandle = jobHandle + self._jobDir = jobHandle.location + + # stdout (line-buffered) + self.stdout = open(self._jobHandle.stdoutFile, "wt", 1) + # stderr (unbuffered) + self.stderr = open(self._jobHandle.stderrFile, "wt", 0) + + def _end(self): + """This is called after run(). + + This is also called when either _begin() or run() raised an exception. + """ + if self.stdout is not None: + self.stdout.close() + self.stdout = None + if self.stderr is not None: + self.stderr.close() + self.stderr = None Added: cgkit/trunk/cgkit/jobqueue/jobqueue.py =================================================================== --- cgkit/trunk/cgkit/jobqueue/jobqueue.py (rev 0) +++ cgkit/trunk/cgkit/jobqueue/jobqueue.py 2009-08-20 20:56:55 UTC (rev 337) @@ -0,0 +1,1032 @@ +# ***** BEGIN LICENSE BLOCK ***** +# Version: MPL 1.1/GPL 2.0/LGPL 2.1 +# +# The contents of this file are subject to the Mozilla Public License Version +# 1.1 (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.mozilla.org/MPL/ +# +# Software distributed under the License is distributed on an "AS IS" basis, +# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License +# for the specific language governing rights and limitations under the +# License. +# +# The Original Code is the Python Computer Graphics Kit. +# +# The Initial Developer of the Original Code is Matthias Baas. +# Portions created by the Initial Developer are Copyright (C) 2009 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# +# Alternatively, the contents of this file may be used under the terms of +# either the GNU General Public License Version 2 or later (the "GPL"), or +# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), +# in which case the provisions of the GPL or the LGPL are applicable instead +# of those above. If you wish to allow use of your version of this file only +# under the terms of either the GPL or the LGPL, and not to allow others to +# use your version of this file under the terms of the MPL, indicate your +# decision by deleting the provisions above and replace them with the notice +# and other provisions required by the GPL or the LGPL. If you do not delete +# the provisions above, a recipient may use your version of this file under +# the terms of any one of the MPL, the GPL or the LGPL. +# +# ***** END LICENSE BLOCK ***** + +import sys, os, os.path, glob, inspect, socket +import random, traceback, shutil +import xml.dom.minidom +import ConfigParser as configparser +from jobproc import JobProc +from jobhandle import JobHandle + +class JobQueueError(Exception): + pass + + +def _convertPyValueToDOM(val, doc): + """Convert a Python value into a DOM element. + + val can be a bool, int, float, str, tuple, list, dict or None. + This is used for writing values into a job definition file. + """ + t = type(val) + typeName = t.__name__ + el = doc.createElement(typeName) + # None? + if val is None: + pass + # Basic type? + elif t in [bool, int, float, str]: + el.setAttribute("value", str(val)) + # Sequence? + elif t in [tuple, list]: + for v in val: + cel = _convertPyValueToDOM(v, doc) + el.appendChild(cel) + # Dict + elif t==dict: + for key,v in val.items(): + keyEl = _convertPyValueToDOM(key, doc) + valEl = _convertPyValueToDOM(v, doc) + el.appendChild(keyEl) + el.appendChild(valEl) + else: + raise ValueError("Unsupport parameter type: %s"%typeName) + + return el + +def _convertDOMToPyValue(element): + """Convert a DOM element into a Python object. + + This is used to convert the parameters from a job definition file + into Python values. + """ + typeName = element.nodeName + + if typeName=="dict": + res = {} + keyValid = False + key = None + for childEl in element.childNodes: + if childEl.nodeType==childEl.ELEMENT_NODE: + v = _convertDOMToPyValue(childEl) + if keyValid: + res[key] = v + keyValid = False + else: + key = v + keyValid = True + if keyValid: + raise ValueError("Dict key without value") + elif typeName=="str": + res = str(element.getAttribute("value")) + elif typeName=="bool": + val = str(element.getAttribute("value")) + res = {"True":True, "False":False}.get(val) + if res is None: + raise ValueError("Invalid boolean value: %s"%val) + elif typeName=="int": + res = int(element.getAttribute("value")) + elif typeName=="float": + res = float(element.getAttribute("value")) + elif typeName in ["list", "tuple"]: + res = [] + for childEl in element.childNodes: + if childEl.nodeType==childEl.ELEMENT_NODE: + res.append(_convertDOMToPyValue(childEl)) + if typeName=="tuple": + res = tuple(res) + elif typeName=="NoneType": + res = None + else: + raise ValueError("Unknown parameter type: %s"%typeName) + return res + + +class _DummyLogger: + """Dummy logger object that does nothing. + """ + def debug(self, *params, **kwargs): + pass + + info = debug + warning = debug + error = debug + critical = debug + log = debug + exception = debug + + +class Job(object): + """Job object (during creation). + + Objects of this class represent jobs that are just being created. + + """ + def __init__(self, jobRoot, jobType, **params): + """Constructor. + + jobRoot is the JobRoot object that this job belongs to. + jobType is a string containing the name of the job class that should + be instantiated. + params are the job parameters which must be passed as keyword + arguments. + """ + if not isinstance(jobRoot, JobRoot): + raise TypeError("jobRoot must be a JobRoot object") + if not isinstance(jobType, basestring): + raise TypeError("Job type must be a string") + + # The root of the job + self._jobRoot = jobRoot + # The job proc class name + self._jobType = jobType + # The job proc parameters + self._params = params + # A list of Job object which this job depends on + self._dependencies = [] + + # This is used later on to detect cycles + self._marked = False + # The maximum distance to a leaf (job nodes without a dependency + # always have a value of 0). This is used to sort the dependencies + # with respect to the longest path in that sub-tree which is necessary + # to remove redundant dependencies. + self._maxLeafDist = 0 + # This is used later on to remove redundant dependencies + self._depNr = 0 + + # The job location on disk. + self._location = self._initJobDir(jobType, **params) + # Indicates whether the job directory is inside the repository or not + # (all jobs begin in the repository except for the root job) + self._isInsideRepository = True + + def createJob(self, jobType, **params): + """Create a new sub-job. + + *jobType* is the name of the job procedure that should be created. Any + additional keyword arguments are passed to the constructor of the job + procedure. + + Returns a :class:Job object that represents the newly created job. + + This method is equivalent to creating a job object manually and + calling addDependency(job). + """ + job = Job(self._jobRoot, jobType, **params) + self.addDependency(job) + return job + + def addDependency(self, job): + """Establish a dependency between another job. + + This job will only be run when all dependencies have been successfully + finished. + """ + if job is self: + raise ValueError("A job cannot depend on itself") + if job._jobRoot!=self._jobRoot: + raise ValueError("A job cannot depend on a job from another job hierarchy") + if isinstance(job, JobRoot): + raise ValueError("A job cannot depend on the job root") + + # Set the dependency + self._dependencies.append(job) + + # TODO: If self has a job dir, then create job dirs for all jobs + # in the job subtree that don't have a job dir yet + + def _initJobDir(self, jobType, **params): + """Create and initialize the job directory. + + jobType is the name of the job procedure that should be created. Any + additional keyword arguments are passed to the constructor of the job + procedure. + Returns the job directory. + """ + + if type(jobType) is not str: + raise TypeError("Job type must be a string") + + # Create an instance of the job procedure to validate the parameters and get the job label + jobProc = self._instantiateJobProc(jobType, **params) + + # Create the root job directory... + jobDir = self._createJobDir() + # This handle is only used for retrieving file locations (so the root is set to None) + jobHandle = JobHandle(jobDir, None) + + # Call the job procedure's postCreate method + jobProc.postCreate(jobDir) + jobProc._jobDir = jobDir + + # Write the label + label = jobProc.label + if label is not None: + f = open(jobHandle.labelFile, "wt") + f.write(label) + f.close() + + # Write the job procedure description. + # If this job node is the job root then write the file under a temporary + # name (so that the job won't get picked up yet) + procDefFile = jobHandle.procDefFile + if self is self._jobRoot: + procDefFile += "_tmp" + self._writeJobDef(procDefFile, jobType, params) + + return jobDir + + def _writeJobDef(self, fileName, jobType, params): + """Write the job definition XML file. + """ + impl = xml.dom.minidom.getDOMImplementation() + doc = impl.createDocument(None, "JobProcedure", None) + jobProc = doc.documentElement + # Set the job type + jobProc.setAttribute("type", jobType) + + # Set the job parameters + paramEl = doc.createElement("parameters") + jobProc.appendChild(paramEl) + paramDictEl = _convertPyValueToDOM(params, doc) + paramEl.appendChild(paramDictEl) + + # Write the XML file + f = open(fileName, "wb") + jobProc.writexml(f, addindent=" ", newl="\n") + f.close() + + jobProc.unlink() + + def _instantiateJobProc(self, jobType, **params): + """Create an instance of a job procedure. + """ + return self._jobRoot._instantiateJobProc(jobType, **params) + + def _createJobDir(self): + """Determine the name of the job directory and create it. + """ + return self._jobRoot._createSubJobDir() + + def _preprocessGraph(self): + """Preprocess the dependency graph. + + Sets the maxLeafDist value on the sub-jobs and checks for cycles. + + Raises a JobQueueError if the graph contains a cycle. + """ + # Have we visited this job already before in this particular path? -> Cycle + if self._marked: + raise JobQueueError("Job hierarchy is impossible to process because it contains a cycle") + + # Mark this job as visited (if we see it again while processing children, + # we have detected a cycle) + self._marked = True + + for subJob in self._dependencies: + # Descend the tree + subJob._preprocessGraph() + self._maxLeafDist = max(self._maxLeafDist, subJob._maxLeafDist+1) + + # Reset the cycle detection mark (as this sub-tree is done and does not contain a cycle) + self._marked = False + + def _writeDependencies(self): + """Write the dependency hierarchy. + + Creates the final job directories (or links) that make up the + dependency hierarchy. + """ + jobDir = self._location + + myDepNr = self._jobRoot._getNextDepNr() + self._depNr = myDepNr + + # Create a dependency list that has the current order associated with each dependency. + # This number will be used as local job number (so that the jobs are + # picked up in the same order as they were added). + deps = [x for x in enumerate(self._dependencies)] + # Sort according to descending maxLeafDist. + # This is done so that the longest paths are processed first and + # redundant dependencies (which always have a shorter path) can be removed. + deps.sort(key=lambda x: -x[1]._maxLeafDist) + + # Create the dependency graph job directories for the children + for jobNr,subJob in deps: + # Do we already have a (possibly indirect) dependency set up? + # Then there's nothing to do anymore + if subJob._depNr>=myDepNr: + continue + + dirName = os.path.join(jobDir, "job%d"%(jobNr)) + if not self._jobRoot._keepJobsInRepository and subJob._isInsideRepository: + # Move the directory from the repository into the dependency hierarchy + shutil.move(subJob._location, dirName) + subJob._location = dirName + subJob._isInsideRepository = False + else: + # Create a link to the actual job dir + self._jobRoot._linkJobDir(subJob._location, dirName) + subJob._setDepNr(myDepNr) + + subJob._writeDependencies() + + def _setDepNr(self, nr): + self._depNr = nr + for j in self._dependencies: + j._setDepNr(nr) + + +class JobRoot(Job): + """The root of a job hierarchy. + + In principle, this is a job like every other job as well, but because + it has a directory inside the main job directory it must remain the + root of the job hierarchy. + The root is also the only job directory that has a special job repository + directory that may store all the sub-jobs. + + This class is derived from the :class:Job class. + """ + + def __init__(self, jobQueue, jobType, **params): + """Constructor. + + jobQueue is the JobQueue object that this job is associated with. + jobType is a string containing the name of the job class that should + be instantiated. + params are the job parameters which must be passed as keyword + arguments. + """ + if not isinstance(jobQueue, JobQueue): + raise TypeError("jobQueue must be a JobQueue object") + self._jobQueue = jobQueue + # This counter is used to create the job directory names in the job repository dir + self._subJobCounter = 0 + # Initial value for the redundant dependency removal part + self._nextDepNr = 0 + # This flag determines whether the jobs will be kept in the job repository + # or moved out into the dependency hierarchy when _writeDependencies() + # is called on any job. + self._keepJobsInRepository = self._jobQueue.keepJobsInRepository + # Can we use sym links or do we have to emulate them? + self._useSymLinks = self._jobQueue.useSymLinks + + Job.__init__(self, self, jobType, **params) + + # Overwrite the isInsideRepository flag (as the root is never inside the repo) + self._isInsideRepository = False + + def activate(self): + """Activate the job so that it can be processed. + + This is the last step of a job submission. After calling this method, + the job hierarchy must not be changed anymore. + + Raises a :exc:JobQueueError if the graph contains a cycle. + """ + # Check for cycles, set the minimum job depth in the graph and sort + # the dependencies of each job according to the minimum depth + self._preprocessGraph() + + # Create the actual job* directories that make up the job hierarchy + self._writeDependencies() + + # Clean up the job repo dir if the jobs have been moved out. + if not self._keepJobsInRepository: + subJobRepo = self._getSubJobRepo() + if os.path.exists(subJobRepo): + # At this point, the directory must be empty, so a single rmdir is enough. + # (it's empty because every job (except for the root job) has at least + # one parent which means the job has been moved out of the repo and + # put under a parent) + os.rmdir(subJobRepo) + + jobHandle = JobHandle(self._location, self._location) + procDefFile = jobHandle.procDefFile + tmpProcDefFile = procDefFile+"_tmp" + if not os.path.exists(tmpProcDefFile): + raise JobQueueError("Failed to activate") + os.rename(tmpProcDefFile, procDefFile) + + def _linkJobDir(self, src, dst): + """Create a link at dst pointing to src. + """ + if not src.startswith(self._location): + raise ValueError("Cannot link directories outside the job root") + + if self._useSymLinks: + # Create a sym link + os.symlink(src, dst) + else: + # Create a file that contains the target path + f = open(dst, "wt") + f.write("[link]\ntarget=$JOBROOT%s\n"%src[len(self._location):]) + f.close() + + def _createSubJobDir(self): + """Create a new directory for a sub-job. + + Note: The sub-job may be anywhere in the final dependency graph. + """ + self._subJobCounter += 1 + # The directory where all sub-jobs are stored + subJobRepo = self._getSubJobRepo() + # Create the repo dir if it doesn't already exist + if not os.path.exists(subJobRepo): + os.mkdir(subJobRepo) + # Create the actual sub-job directory + subJobDir = os.path.join(self._location, ".jobs", "j%d"%self._subJobCounter) + os.mkdir(subJobDir) + return subJobDir + + def _getNextDepNr(self): + self._depNr += 1 + return self._depNr + + def _getSubJobRepo(self): + """Return the location of the job repository directory. + """ + return os.path.join(self._location, ".jobs") + + def _instantiateJobProc(self, jobType, **params): + """Create an instance of a job procedure. + """ + return self._jobQueue._instantiateJobProc(jobType, **params) + + def _createJobDir(self): + """Determine the name of the job directory and create it. + """ + # Create the root job directory... + for i in range(10): + jobDir = self._jobQueue._newJobDir() + try: + os.mkdir(jobDir) + break + except OSError: + pass + else: + raise JobQueueError("Failed to create a new job directory") + + return jobDir + + +class JobQueue: + """Job queue class. + + A job queue contains a list of jobs which can be run by one or more + processes. Each job may be composed of sub-jobs which all have to + be successfully completed before the job is allowed to be processed. + Jobs that are on the same level in the hierarchy are independent and + can be run concurrently. + + An individual job is represented by a job procedure (JobProc) object + that implements the actual functionality of a particular type of job. + A job is considered done when the run() method of a job procedure has + been executed successfully and the job procedure did not indicate an + error. + + On disk, the entire job queue is stored as a directory which contains + a job directory for each job in the queue. Sub-jobs are sub-directories + of the job directories. + """ + + def __init__(self, location, logger=None): + """Constructor. + + location is a string containing the directory where the job queue + is stored (the directory should already exist, it is not created). + It is also valid to pass None (which is equivalent to an empty + queue that cannot receive jobs). + + logger can be set to a logger object from the logging module (or + an object with the same interface) which will receive log message. + """ + # Verify the input types + if location is not None and not isinstance(location, basestring): + raise TypeError("Job location must be a string or None.") + + # Make the directory path absolute + if location is not None: + location = os.path.abspath(location) + if not os.path.exists(location): + raise ValueError("Job queue %s does not exist"%location) + + if logger is None: + logger = _DummyLogger() + + if location is not None: + logger.info("Attach to job queue at %s"%location) + + self._logger = logger + self._location = location + self.keepJobsInRepository = False + self.useSymLinks = False + + # The default parameter values for job procedures. + # Key:Proc name - Value:Keyword params dict + self._defaultProcParams = {} + + self._nextDepNr = 0 + + # Read the config file + if self._location is not None: + configName = os.path.join(self._location, "queue.cfg") + self._readConfigFile(configName) + + # Import the proc classes... + self._procClasses = {} + if self._location is not None: + origPaths = sys.path[:] + procDir = os.path.join(self._location, "procs") + pyNames = glob.glob(os.path.join(procDir, "*.py")) + sys.path.insert(0, procDir) + for pyName in pyNames: + modName = os.path.splitext(os.path.basename(pyName))[0] + ns = {} + exec "import %s"%modName in ns + if hasattr(ns[modName], modName): + cls = getattr(ns[modName], modName) + if inspect.isclass(cls) and cls!=JobProc and issubclass(cls, JobProc): + self._procClasses[modName] = cls + sys.path = origPaths + + @property + def location(self): + """Return the absolute location of the job queue directory. + + Returns None if no directory is set. + """ + return self._location + + def listJobs(self): + """Return a list of all top-level jobs. + + Returns a list of :class:JobHandle objects that represent the + roots of all jobs. + The order of the jobs is in the order they would be processed. + If the job queue is not associated with a job directory, the return + value is an empty list. + """ + if self._location is None: + return [] + + # List all job* sub-directories + jobDirPattern = os.path.join(self._location, "job*") + jobDirs = glob.glob(jobDirPattern) + #jobDirs = filter(lambda p: os.path.isdir(p), jobDirs) + jobs = map(lambda jobDir: (jobDir, int(os.path.basename(jobDir)[3:])), jobDirs) + jobs.sort(key=lambda a: a[1]) + return [JobHandle(jobDir, jobDir) for jobDir,nr in jobs] + + def createJobRoot(self, jobType=None, **params): + """Create a new top-level job. + + *jobType* is the name of the job procedure that should be created. Any + additional keyword arguments are passed to the constructor of the job + procedure. + + Returns a :class:JobRoot object that + represents the newly created job. + Sub-jobs can be created by calling :meth:createJob() + on the returned job root. + + After the entire job hierarchy has been created, the + :meth:activate() + method must be called on the job root, otherwise the job will not + be processed. + """ + jobRoot = JobRoot(self, jobType, **params) + return jobRoot + + # An alias for createJobRoot (for single command jobs) + createJob = createJobRoot + + def deleteJobs(self, jobs): + """Delete one or more jobs. + + *jobs* is either a single :class:JobHandle object + or a sequence of :class:JobHandle objects. + *jobs* may also be None in which case the method returns immediately. + + The method raises an error (:exc:OSError) if it couldn't delete a job. + This will abort the entire operation and some of the input jobs + may not even have been processed. + Jobs that don't exist at all anymore are not considered an error + and won't trigger an exception. + + Note: The method is meant to be used on top-level jobs. Currently, + it can also be used for sub-jobs, but it doesn't check if deleting + a sub-job breaks another sub-job that references a job within the + deleted job. + """ + if jobs is None: + return + if isinstance(jobs, JobHandle): + jobs = [jobs] + + for job in jobs: + if os.path.exists(job.location): + self._logger.info("Deleting job directory %s"%job.location) + # Delete the proc def file first, so that nobody can pick up this job anymore + try: + os.remove(job.procDefFile) + except: + pass + # Now delete everything + shutil.rmtree(job.location) + + def runNextAvailableJob(self, retries=10): + """Run the next available job in the queue that is in a waiting state. + + The return value is True if a job could be run or False if there + are no more waiting jobs at the moment. + When there are waiting jobs but the method fails to allocate one of them, + then the integer *retries* determines how many attempts should be made + before giving up. Once the last attempt has failed, a :exc:JobQueueError + exception is thrown. Some causes for this could be: + + - The process doesn't have file access permissions for the job directory + - There were too many other processes that were slightly quicker in + getting permission + - There is a broken job directory somewhere (if it's in waiting state + but can't be run for some reason (maybe the proc is missing)) + """ + for i in range(retries): + # Get a list of all available jobs + jobs = self.listJobs() + # Pick the next available job + job = self._findNextWaitingJob(jobs) + if job is None: + return False + # Try to run it + if self._runJob(job): + return True + + raise JobQueueError("Failed to get permission to run a job") + + def _findNextWaitingJob(self, jobs): + """Find the next job that should be processed. + + jobs is a sequence of JobHandle objects. The method returns a JobHandle + object that is the "deepest" job that is in waiting state. + """ + # Ignore any job that is not currently waiting + jobs = filter(lambda job: job.isWaiting(), jobs) + + # Search for the first waiting sub-job... + for job in jobs: + subJobs = job.listSubJobs() + if self._isReady(subJobs): + return job + + # The job is not ready, so try to pick one of the sub-jobs... + j = self._findNextWaitingJob(subJobs) + if j is not None: + return j + + return None + + def _isReady(self, subJobs): + """Check if a job with the given sub-jobs is ready to run. + + subJobs is a sequence of JobHandle objects (obtained from job.listSubJobs()). + Returns True when all jobs in subJobs have been finished without + an error. + """ + for job in subJobs: + if not job.isFinished() or job.hasError(): + return False + return True + + def _runJob(self, job): + """Try to run the job. + + job is a JobHandle object representing the job to run. + The return value is True if the job could be run, otherwise the + value is False. A value of False means, this process couldn't bring + the job into running state (e.g. because it was not in waiting state + or another process was slightly quicker to run it or there were + permission issues). It is absolutely possible that the method returns + False even when a previous call to isWaiting() returned True. + + Note: A return value of True only means this process got permission + to run the job. It doesn't mean that the job also completed successfully. + + It is the callers responsibility to make sure that all sub-jobs + have completed successfully. + """ + self._logger.info("Trying to run job %s"%job.location) + # Read the job procedure first + try: + jobType,params = self._readJobDef(job.procDefFile) + except: + self._logger.info("Failed to read job procedure definition: %s"%sys.exc_info()[1]) + return False + + # Apply the default parameter values from the config file + defaultParams = self._defaultProcParams.get(jobType, {}) + for name,val in defaultParams.items(): + if name not in params: + params[name] = val + + # Create a new instance of the job procedure + proc = self._instantiateJobProc(jobType, **params) + + # Try to bring the job into running state + if not self._setJobToRunningState(job): + return False + + # Run the job procedure + self._logger.info("Running job") + os.chdir(job.runningDir) + try: + proc._begin(job) + proc.run() + self._logger.info("Job succeeded") + except: + self._logger.info("Job failed to run successfully") + job.setError() + f = open(job.procTracebackFile, "wt") + traceback.print_exc(file=f) + f.close() + try: + proc._end() + except: + pass + + # Mark the job as being finished + self._logger.debug("Creating finished directory %s"%job.finishedDir) + try: + os.mkdir(job.finishedDir, 0777) + except: + self._logger.warn("Failed to create directory %s: "%(job.finishedDir, sys.exc_info()[1])) + + return True + + def _setJobToRunningState(self, job): + """Mark a job as being run by this process. + + Returns True if the job could be allocated (i.e. brought into running + state) or False if we are not allowed to run the job. + """ + runningDir = job.runningDir + pidFile = job.pidFile + hostFile = job.hostFile + + # Try to create the ".running" directory. If it fails, we assume + # it exists already and return. Otherwise we have permission to run + # the job procedure + self._logger.debug("Creating %s"%runningDir) + try: + os.mkdir(runningDir, 0777) + except OSError, exc: + self._logger.debug("Failed to create directory: %s"%exc) + return False + + # Write the info files into the .running dir + try: + # Store the process id of this process inside the directory + pid = os.getpid() + f = open(pidFile, "wt") + try: + print >>f, pid + finally: + f.close() + + # Store the host name inside the directory + host = socket.gethostname() + f = open(hostFile, "wt") + try: + print >>f, host + finally: + f.close() + except: + err = sys.exc_info()[1] + self._logger.warning("Failed to write pid/host file: %s"%err) + # Something went wrong while trying to write the pid and host file. + # So try to remove the .running directory again and return False. + try: + os.remove(pidFile) + except: + pass + try: + os.remove(hostFile) + except: + pass + try: + os.rmdir(runningDir) + except: + pass + return False + + return True + + + def _readJobDef(self, fileName): + """Read the job definition. + + Returns the job type string and the parameter dict. + Raises an error when there was an error reading the file. + """ + doc = xml.dom.minidom.parse(fileName) + + jobProc = doc.documentElement + if jobProc.nodeName!="JobProcedure": + raise ValueError("Error in job definition file, expected a job procedure element") + + jobType = jobProc.getAttribute("type") + if jobType=="": + raise ValueError("Error in job definition file. No job type given.") + + params = jobProc.getElementsByTagName("parameters") + if len(params)==0: + raise ValueError("Error in job definition file. No job parameters found.") + if len(params)>1: + raise ValueError("Error in job definition file. Multiple parameter elements found.") + params = params[0] + + paramDict = jobProc.getElementsByTagName("dict") + if len(paramDict)!=1: + raise ValueError("Error in job definition file. Invalid parameter element.") + paramDict = paramDict[0] + + params = _convertDOMToPyValue(paramDict) + + return jobType, params + + def _readConfigFile(self, fileName): + """Read the job queue config file. + + Updates the local attributes that are initialized from the config file. + """ + self._logger.info("Read config file %s"%fileName) + cp = configparser.ConfigParser() + # Override the optionxform method so that it returns the input string + # unmodified. We need case-senstivity as some options are actually + # argument names for the job procedures. + cp.optionxform = str + cp.read(fileName) + + # This dictionary defines the valid options that may appear in the + # [main] section. The values are the default values which also + # define the valid type of the variable. + cfgDict = {"keepJobsInRepository":self.keepJobsInRepository, + "useSymLinks":self.useSymLinks} + + # Check if there are unknown options on the main section and issue + # warnings if there are... + if cp.has_section("main"): + knownOptNames = {} + for key in cfgDict.keys(): + knownOptNames[key.lower()] = 1 + opts = cp.options("main") + for opt in opts: + if opt.lower() not in knownOptNames: + self._logger.warning('Unknown job queue config variable "%s" in file %s\n'%(opt, fileName)) + + # Get the values from the main section. The config variable values + # are set as local attributes. + for key in cfgDict.keys(): + defaultVal = cfgDict[key] + val = defaultVal + if cp.has_option("main", key): + if type(defaultVal) is bool: + val = cp.getboolean("main", key) + else: + raise TypeError("Internal error: Unknown config var type") + setattr(self, key, val) + + # Read the proc defaults + for section in cp.sections(): + if section.endswith(" proc"): + defaultParams = {} + procName = section[:-5] + for option in cp.options(section): + val = cp.get(section, option) + defaultParams[option] = val + self._defaultProcParams[procName] = defaultParams + + def _instantiateJobProc(self, jobType, **params): + """Create an instance of a job procedure. + """ + if jobType not in self._procClasses: + raise ValueError("Unknown job type '%s'"%jobType) + + cls = self._procClasses[jobType] + return cls(**params) + + def _newJobDir(self): + """Return a full new potential job directory name. + + The returned directory name may already be taken if another process + has picked exactly the same name at almost the same time. The returned + names contain some randomness to reduce the likeliness of such a + name clash. + """ + # Determine the largest existing job number + jobs = self.listJobs() + if len(jobs)==0: + last = 0 + else: + lastJobLocation = jobs[-1].location + # Jobs are named "jobX" where X is the job number, so strip + # off the "job" part of the name to get the job number back. + last = int(os.path.basename(lastJobLocation)[3:]) + + # The next job number will be at least this much higher (must be >0) + # The higher this value, the more jobs fit in-between this job and + # the previous job in case, priorities are modified later on. + minDelta = 100 + # The next job number will at most be this much higher (must be >=minDelta) + # The difference between the min and max delta should at least be about + # the number of submissions that are expected to be done simultaneously. + maxDelta = 1000 + nr = random.randint(last+minDelta, last+maxDelta) + jobDir = os.path.join(self._location, "job%d"%nr) + return jobDir + +def createJobQueue(location, keepJobsInRepository=False, useSymLinks=False): + """Create and initialize a new job queue directory. + + *location* is a string containing the directory path where the job + queue should be put. This must either refer to an empty directory or + to a path that does not exist (the parent must exist though). + + When jobs are created, they are first stored in a dedicated job repository + directory (within the job queue directory). The *keepJobsInRepository* + flag determines whether the jobs will be moved into their respective + parent job directory or whether they remain in the repository directory. + + *useSymLinks* determines whether sub-jobs are linked by creating + symbolic links on disk or by creating files that contain the target + directory. This can only be set to True if the file system actually + supports symbolic links (so on Windows this should always be False). + The only advantage of using symbolic links is that it will be easier + to navigate the job directory manually. However, the disadvantage is that + the entire job queue directory will not be location independent anymore. + + Once the job queue has been created successfully, you can queue jobs + using the :class:JobQueue class. + """ + # Check the type of location + if not isinstance(location, basestring): + raise TypeError("Job location must be a string.") + + # If the directory exists, check that it is empty, otherwise create it. + if os.path.exists(location): + dp,dirNames,fileNames = os.walk(location).next() + if dirNames!=[] or fileNames!=[]: + raise ValueError("Cannot create a new job queue in '%s'. The directory is not empty."%location) + else: + # Create the root job queue directory + os.mkdir(location) + + # Create the procs directory (only writable for the current user) + procPath = os.path.join(location, "procs") + os.mkdir(procPath, 0755) + + # Copy the default procedure scripts into the procs directory. + # The permissions of the scripts are set to -r--r--r-- (to prevent + # accidental deletion or modification) + procTemplatePath = os.path.join(os.path.dirname(__file__), "defaultprocs") + procs = glob.glob(os.path.join(procTemplatePath, "*.py")) + for proc in procs: + # Skip __init__.py + if os.path.basename(proc).startswith("__"): + continue + dst = os.path.join(procPath, os.path.basename(proc)) + shutil.copy(proc, dst) + os.chmod(dst, 0444) + + # Create the config file + configPath = os.path.join(location, "queue.cfg") + f = open(configPath, "wt") + f.write("# Job Queue config file\n\n") + f.write("[main]\n\n") + f.write("KeepJobsInRepository = %s\n"%bool(keepJobsInRepository)) + f.write("UseSymLinks = %s\n"%bool(useSymLinks)) + f.close() Modified: cgkit/trunk/setup.py =================================================================== --- cgkit/trunk/setup.py 2009-08-15 12:02:09 UTC (rev 336) +++ cgkit/trunk/setup.py 2009-08-20 20:56:55 UTC (rev 337) @@ -301,7 +301,8 @@ "utilities/convert3d.py", "utilities/seqls.py", "utilities/seqmv.py", - "utilities/seqrm.py",] + "utilities/seqrm.py", + "utilities/seqcp.py"] # Lib3ds if LIB3DS_AVAILABLE: @@ -658,7 +659,10 @@ PACKAGE_NAME+".Interfaces", PACKAGE_NAME+".wintab", PACKAGE_NAME+".spacedevice", - PACKAGE_NAME+".glove"], + PACKAGE_NAME+".glove", + PACKAGE_NAME+".jobqueue", + PACKAGE_NAME+".jobqueue.defaultprocs", + ], # Stick all extension modules into the main package ext_package = PACKAGE_NAME, Added: cgkit/trunk/unittests/test_jobqueue.py =================================================================== --- cgkit/trunk/unittests/test_jobqueue.py (rev 0) +++ cgkit/trunk/unittests/test_jobqueue.py 2009-08-20 20:56:55 UTC (rev 337) @@ -0,0 +1,83 @@ +# Test the jobqueue module + +import unittest, os.path +import sys, shutil +from cgkit.jobqueue import createJobQueue, JobQueue, JobQueueError +from cgkit.jobqueue.jobhandle import JobHandle + +class TestJobQueue(unittest.TestCase): + + def testMultiRef(self): + jq = self.queue() + jr = jq.createJobRoot("blender", blendFile="multiple_references.blend") + shad = jr.createJob("blender", blendFile="shadow_map") + j1 = jr.createJob("blender", blendFile="frame1") + j2 = jr.createJob("blender", blendFile="frame2") + j1.addDependency(shad) + j2.addDependency(shad) + jr.activate() + print jr._location + + def testDependencySimplification(self): + """Check that redundant dependencies are removed. + """ + jq = self.queue() + jr = jq.createJobRoot("blender", blendFile="check_redundant_deps.blend") + j1 = jr.createJob("blender", blendFile="foo1") + j2 = jr.createJob("blender", blendFile="foo2") + # The direct dependency between jr and j1 should be removed again + # (because jr depends on j2 and j2 on j1, so jr indirectly depends on j1 anyway) + j2.addDependency(j1) + jr.activate() + + jh = JobHandle(jr._location, jr._location) + subJobs = list(jh.listSubJobs()) + self.assertEqual(1, len(subJobs)) + self.assertEqual('Blender "foo2"', subJobs[0].label) + subSubJobs = list(subJobs[0].listSubJobs()) + self.assertEqual(1, len(subSubJobs)) + self.assertEqual('Blender "foo1"', subSubJobs[0].label) + + # Do the same as above but reverse the order of the dependency + jr = jq.createJobRoot("blender", blendFile="check_redundant_deps.blend") + j1 = jr.createJob("blender", blendFile="foo1") + j2 = jr.createJob("blender", blendFile="foo2") + # The direct dependency between jr and j2 should be removed again + # (because jr depends on j1 and j1 on j2, so jr indirectly depends on j2 anyway) + j1.addDependency(j2) + jr.activate() + + jh = JobHandle(jr._location, jr._location) + subJobs = list(jh.listSubJobs()) + self.assertEqual(1, len(subJobs)) + self.assertEqual('Blender "foo1"', subJobs[0].label) + subSubJobs = list(subJobs[0].listSubJobs()) + self.assertEqual(1, len(subSubJobs)) + self.assertEqual('Blender "foo2"', subSubJobs[0].label) + + def testCycleDetection(self): + """Check if cycles in the dependency graph are detected. + """ + jq = self.queue() + jr = jq.createJobRoot("blender", blendFile="check_cycle_detection.blend") + j1 = jr.createJob("blender", blendFile="foo1") + j2 = j1.createJob("blender", blendFile="foo2") + j3 = j2.createJob("blender", blendFile="foo3") + j3.addDependency(j1) + self.assertRaises(JobQueueError, lambda: jr.activate()) + + def queue(self): + """Return a JobQueue object. + """ + if not os.path.exists("tstqueue"): + createJobQueue("tstqueue") + return JobQueue("tstqueue") + +###################################################################### + +# Remove the queue from a previous run +if os.path.exists("tstqueue"): + shutil.rmtree("tstqueue") + +if __name__=="__main__": + unittest.main() This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. `