[Pyphant-devel] SF.net SVN: pyphant:[691] trunk/src/workers/fmfile/fmfile/FMFLoader.py

[<zk...@us...>]

Revision: 691
          http://pyphant.svn.sourceforge.net/pyphant/?rev=691&view=rev
Author:   zklaus
Date:     2010-06-30 11:17:29 +0000 (Wed, 30 Jun 2010)

Log Message:
-----------
Merge branch 'master' into svn-trunk

* master:
  Enh: FMFLoader speedup

Modified Paths:
--------------
    trunk/src/workers/fmfile/fmfile/FMFLoader.py

Modified: trunk/src/workers/fmfile/fmfile/FMFLoader.py
===================================================================
--- trunk/src/workers/fmfile/fmfile/FMFLoader.py	2010-06-29 12:01:34 UTC (rev 690)
+++ trunk/src/workers/fmfile/fmfile/FMFLoader.py	2010-06-30 11:17:29 UTC (rev 691)
@@ -74,6 +74,7 @@
         else:
             data[pixelName] = rawContainer
         subscriber %= float(i+1)/total*100.0
+
     return data,names
 
 def collectAttributes(data,names):
@@ -145,7 +146,7 @@
                 indexError = 2
                 for i,element in enumerate(column):
                     if not type(element)==type((0,)):
-                        column[i]=(shortname,numpy.NaN,None) 
+                        column[i]=(shortname,numpy.NaN,None)
             try:
                 data = [element[indexDatum] for element in column]
             except:
@@ -257,35 +258,13 @@
     if field.isIndependent():
         return field
     dimData = [numpy.unique(d.data) for d in field.dimensions]
-    fieldData = numpy.ones([len(d) for d in dimData])*numpy.nan
-    data = numpy.vstack([field.data]+[d.data for d in field.dimensions]).transpose()
-    for row in data:
-        try:
-            fieldData[[numpy.argwhere(dimData[i]==v) for i,v in enumerate(row[1:])]] = row[0]
-        except AttributeError:
-            from pyphant.wxgui2.wxPyphantApplication import LOGDIR
-            import os, os.path
-            DEBDIR=os.path.join(LOGDIR, "pyphant_debug")
-            if not os.path.exists(DEBDIR):
-                os.mkdir(DEBDIR)
-            for i,v in enumerate(row[1:]):
-                try:
-                    numpy.argwhere(dimData[i]==v)
-                except AttributeError, e:
-                    _logger.debug(u"AttributeError occured:",
-                                  exc_info=True)
-                    f = open(os.path.join(DEBDIR, "deblog"), 'w')
-                    f.write("dShape: %s; vShape: %s\n\n"%(dimData[i].shape, v.shape))
-                    f.write("%s"%dimData[i])
-                    f.write("\n\nv:\n")
-                    f.write("%s"%v)
-                    f.write("\n")
-                    f.close()
-                    numpy.savetxt(os.path.join(DEBDIR, "dimData.txt"), dimData[i])
-                    numpy.savetxt(os.path.join(DEBDIR, "data.txt"), data)
-                    numpy.savetxt(os.path.join(DEBDIR, "row.txt"), row)
-                    numpy.savetxt(os.path.join(DEBDIR, "v.txt"), v)
-                    raise
+    dimDicts = [dict([(data, index) for index, data in enumerate(dimdata)]) \
+                for dimdata in dimData]
+    fieldData = numpy.ones([len(d) for d in dimData]) * numpy.nan
+    indicess = [map(lambda x: dimDicts[index][x], dim.data) \
+                for index, dim in enumerate(field.dimensions)]
+    for datum, indices in zip(field.data, indicess):
+        fieldData[indices] = datum
     newDims = [ DataContainer.FieldContainer(dimData[i],
                                              f.unit,
                                              longname=f.longname,
@@ -381,7 +360,6 @@
     return oldVal
 
 def config2tables(preParsedData, config, FMFversion='1.1'):
-
     if config.has_key('*table definitions'):
         longnames = dict([(i,k) for k,i in config['*table definitions'].iteritems()])
         del config['*table definitions']
@@ -399,7 +377,7 @@
                                      preParsedData[shortname],
                                      config[k],FMFversion))
             del config[k]
-    attributes = config.walk(lambda section,key: 
+    attributes = config.walk(lambda section,key:
                              item2value(section[key],FMFversion))
     for t in tables:
         t.attributes = copy.deepcopy(attributes)


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.