Menu
▾
▴
[Pyphant-devel] SF.net SVN: pyphant:[629] trunk/src
|
From: <zk...@us...> - 2009-04-02 16:38:46
|
Revision: 629
http://pyphant.svn.sourceforge.net/pyphant/?rev=629&view=rev
Author: zklaus
Date: 2009-04-02 16:38:39 +0000 (Thu, 02 Apr 2009)
Log Message:
-----------
Merge branch 'master' into svn-trunk
Modified Paths:
--------------
trunk/src/pyphant/pyphant/core/FieldContainer.py
trunk/src/pyphant/pyphant/core/PyTablesPersister.py
trunk/src/workers/ImageProcessing/ImageProcessing/DiffWorker.py
trunk/src/workers/fmfile/fmfile/FMFWriter.py
trunk/src/workers/fmfile/fmfile/fmfgen.py
Added Paths:
-----------
trunk/src/pyphant/pyphant/core/KnowledgeManager.py
trunk/src/pyphant/pyphant/core/knowledge-ideas.org
trunk/src/pyphant/pyphant/tests/TestKnowledgeManager.py
Modified: trunk/src/pyphant/pyphant/core/FieldContainer.py
===================================================================
--- trunk/src/pyphant/pyphant/core/FieldContainer.py 2009-04-02 13:39:35 UTC (rev 628)
+++ trunk/src/pyphant/pyphant/core/FieldContainer.py 2009-04-02 16:38:39 UTC (rev 629)
@@ -411,7 +411,7 @@
_logger.debug('The errors differ: The error of the second argument is none, while the error of the first argument is %s.' % error)
return False
if not numpy.allclose(scaledError,otherScaledError,rtol,atol):
- _logger.debug('The errors differ: %s\n%s' % (scaledError,otherScaledError))
+ _logger.debug('The normed errors differ: %s\n%s' % (scaledError,otherScaledError))
return False
else:
if not data.dtype.char in ['S','U']:
@@ -489,7 +489,7 @@
def __sub__(self, other):
if isinstance(other, FieldContainer):
if self.error!=None or other.error!=None:
- return NotImplemented
+ error = other.error + self.error
else:
error = None
if len(self._dimensions) != len(other.dimensions):
Added: trunk/src/pyphant/pyphant/core/KnowledgeManager.py
===================================================================
--- trunk/src/pyphant/pyphant/core/KnowledgeManager.py (rev 0)
+++ trunk/src/pyphant/pyphant/core/KnowledgeManager.py 2009-04-02 16:38:39 UTC (rev 629)
@@ -0,0 +1,144 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2006-2008, Rectorate of the University of Freiburg
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of the Freiburg Materials Research Center,
+# University of Freiburg nor the names of its contributors may be used to
+# endorse or promote products derived from this software without specific
+# prior written permission.
+#
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+u"""
+knowledge manager
+
+- retrieve data from local HDF5 files for given emd5s
+"""
+
+__id__ = "$Id$"
+__author__ = "$Author$"
+__version__ = "$Revision$"
+# $Source: $
+
+from pyphant.core.singletonmixin import Singleton
+from pyphant.core.DataContainer import parseId
+import pyphant.core.PyTablesPersister as ptp
+
+from types import TupleType
+import urllib
+import tables
+import os, os.path
+import logging
+
+class KnowledgeManagerException(Exception):
+ def __init__(self, message, parent_excep=None, *args, **kwds):
+ super(KnowledgeManagerException, self).__init__(message, *args, **kwds)
+ self._message = message
+ self._parent_excep = parent_excep
+
+ #def __repr__(self):
+ # return message+" (reason: %s)" % (str(self._parent_excep),)
+
+class KnowledgeManager(Singleton):
+
+ def __init__(self):
+ super(KnowledgeManager, self).__init__()
+ self._logger = logging.getLogger("pyphant")
+ self._refs = {}
+
+ def _retrieveURL(self, url):
+ # exceptions?
+ self._logger.info("Retrieving url '%s'..." % (url,))
+ localfilename, headers = urllib.urlretrieve(url)
+ self._logger.info("Using local file '%s'." % (localfilename,))
+ self._logger.info("Header information: %s", (str(headers),))
+
+ #
+ # Save index entries
+ #
+ h5 = tables.openFile(localfilename)
+ # title of 'result_' groups has id in TITLE attribute
+ dc = None
+ for group in h5.walkGroups(where="/results"):
+ id = group._v_attrs.TITLE
+ if len(id)>0:
+ self._logger.debug("Registering id '%s'.." % (id,))
+ self._refs[id] = (url, localfilename, group._v_pathname)
+
+ return localfilename
+
+ def registerURL(self, url):
+ localfilename = self._retrieveURL(url)
+
+ def registerDataContainer(self, datacontainer):
+ try:
+ assert datacontainer.id is not None
+ self._refs[datacontainer.id] = datacontainer
+ except Exception, e:
+ raise KnowledgeManagerException("Invalid id for DataContainer '" +\
+ datacontainer.longname+"'", e)
+
+ #def searchAndRegisterKnowledgeManager(self, host)
+ # KM =remote object auf host
+ # registerKnowledgeManager(self, KM)
+ #def registerKnowledgeManager(self, KM)
+
+ def getDataContainer(self, id, try_cache=True):
+ if id not in self._refs.keys():
+ raise KnowledgeManagerException("Id '%s' unknown." % (id,))
+
+ ref = self._refs[id]
+ if isinstance(ref, TupleType):
+ dc = self._getDCfromURLRef(id, try_cache = try_cache)
+ else:
+ dc = ref
+
+ return dc
+
+ def _getDCfromURLRef(self, id, try_cache=True):
+ url, localfilename, h5path = self._refs[id]
+ if not try_cache:
+ os.remove(localfilename)
+
+ if not os.path.exists(localfilename):
+ localfilename = self._retrieveURL(url)
+ url, localfilename, h5path = self._refs[id]
+
+ h5 = tables.openFile(localfilename)
+
+ hash, type = parseId(id)
+ assert type in ['sample','field']
+ if type=='sample':
+ loader = ptp.loadSample
+ elif type=='field':
+ loader = ptp.loadField
+ else:
+ raise KnowledgeManagerException("Unknown result type '%s'" \
+ % (type,))
+ try:
+ self._logger.debug("Loading data from '%s' in file '%s'.." % (localfilename, h5path))
+ dc = loader(h5, h5.getNode(h5path))
+ except Exception, e:
+ raise KnowledgeManagerException("Id '%s' known, but cannot be read from file '%s'." \
+ % (id,localfilename), e)
+ return dc
Modified: trunk/src/pyphant/pyphant/core/PyTablesPersister.py
===================================================================
--- trunk/src/pyphant/pyphant/core/PyTablesPersister.py 2009-04-02 13:39:35 UTC (rev 628)
+++ trunk/src/pyphant/pyphant/core/PyTablesPersister.py 2009-04-02 16:38:39 UTC (rev 629)
@@ -152,29 +152,6 @@
columns.append(saveResult(column,h5))
h5.setNodeAttr(resultGroup, "columns", columns)
-def loadSample(h5, resNode):
- result = DataContainer.SampleContainer.__new__(DataContainer.SampleContainer)
- result.longname = unicode(h5.getNodeAttr(resNode, "longname"), 'utf-8')
- result.shortname = unicode(h5.getNodeAttr(resNode, "shortname"), 'utf-8')
- result.attributes = {}
- for key in resNode._v_attrs._v_attrnamesuser:
- if key not in _reservedAttributes:
- result.attributes[key]=h5.getNodeAttr(resNode,key)
- columns = []
- for resId in h5.getNodeAttr(resNode,"columns"):
- nodename = "/results/"+resId
- hash, uriType = DataContainer.parseId(h5.getNodeAttr(nodename, "TITLE"))
- if uriType == 'sample':
- loader = loadSample
- elif uriType =='field':
- loader = loadField
- else:
- raise KeyError, "Unknown UriType %s in saving result %s." % (uriType, result.id)
- columns.append(loader(h5,h5.getNode(nodename)))
- result.columns=columns
- result.seal(resNode._v_title)
- return result
-
def saveField(h5, resultGroup, result):
def dump(inputList):
def conversion(arg):
@@ -313,4 +290,26 @@
result.seal(resNode._v_title)
return result
+def loadSample(h5, resNode):
+ result = DataContainer.SampleContainer.__new__(DataContainer.SampleContainer)
+ result.longname = unicode(h5.getNodeAttr(resNode, "longname"), 'utf-8')
+ result.shortname = unicode(h5.getNodeAttr(resNode, "shortname"), 'utf-8')
+ result.attributes = {}
+ for key in resNode._v_attrs._v_attrnamesuser:
+ if key not in _reservedAttributes:
+ result.attributes[key]=h5.getNodeAttr(resNode,key)
+ columns = []
+ for resId in h5.getNodeAttr(resNode,"columns"):
+ nodename = "/results/"+resId
+ hash, uriType = DataContainer.parseId(h5.getNodeAttr(nodename, "TITLE"))
+ if uriType == 'sample':
+ loader = loadSample
+ elif uriType =='field':
+ loader = loadField
+ else:
+ raise KeyError, "Unknown UriType %s in saving result %s." % (uriType, result.id)
+ columns.append(loader(h5,h5.getNode(nodename)))
+ result.columns=columns
+ result.seal(resNode._v_title)
+ return result
Added: trunk/src/pyphant/pyphant/core/knowledge-ideas.org
===================================================================
--- trunk/src/pyphant/pyphant/core/knowledge-ideas.org (rev 0)
+++ trunk/src/pyphant/pyphant/core/knowledge-ideas.org 2009-04-02 16:38:39 UTC (rev 629)
@@ -0,0 +1,29 @@
+
+
+Ideas:
+
+*
+ Before caching an entire file, test whether a KnowledgeManager runs
+ on the remote side. If yes, ask for the file.
+
+* Download auf Wunsch erzwingen, falls remote Datei ge\xE4ndert?
+ Alternativ: KnowledgeManager Cache l\xF6schen lassen, siehe urllib.urlcleanup()
+
+* Was tun, wenn id nicht auffindbar?
+
+ - anderen KnowledgeManager fragen, der sich
+ entweder aus der URL ergibt oder
+ der extra registriert ist
+
+* Referenzen (Index) innerhalb HDF5-Dateien auffrischen?
+
+* Dictionaries/Caches
+
+ - id -> Referenzen (URLs)
+ - Referenzen -> lokale Kopien (urllib macht das schon!)
+ - id -> (Referenz, Knotenreferenz)
+
+
+* Cache Verzeichnis
+Bei Initialisieren des KnowledgeManagers Cache Verzeichnis angeben und
+bei urlretreive nutzen
Added: trunk/src/pyphant/pyphant/tests/TestKnowledgeManager.py
===================================================================
--- trunk/src/pyphant/pyphant/tests/TestKnowledgeManager.py (rev 0)
+++ trunk/src/pyphant/pyphant/tests/TestKnowledgeManager.py 2009-04-02 16:38:39 UTC (rev 629)
@@ -0,0 +1,144 @@
+#!/usr/bin/env python2.5
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2006-2008, Rectorate of the University of Freiburg
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of the Freiburg Materials Research Center,
+# University of Freiburg nor the names of its contributors may be used to
+# endorse or promote products derived from this software without specific
+# prior written permission.
+#
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+# OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+u"""Provides unittest classes
+"""
+
+__id__ = "$Id$".replace('$','')
+__author__ = "$Author$".replace('$','')
+__version__ = "$Revision$".replace('$','')
+# $Source$
+
+import unittest
+import pkg_resources
+pkg_resources.require("pyphant")
+
+from pyphant.core.KnowledgeManager import KnowledgeManager
+import pyphant.core.PyTablesPersister as ptp
+from pyphant.core.FieldContainer import FieldContainer
+import numpy as N
+import tables
+import urllib
+
+import tempfile
+import os
+import logging
+
+
+class KnowledgeManagerTestCase(unittest.TestCase):
+ def setUp(self):
+ a = N.array([0, 1, 2, 3])
+ self._fc = FieldContainer(a)
+ self._fc.seal()
+
+ def testGetLocalFile(self):
+
+ h5fileid, h5name = tempfile.mkstemp(suffix='.h5',prefix='test-')
+ os.close(h5fileid)
+
+ h5 = tables.openFile(h5name,'w')
+ resultsGroup = h5.createGroup("/", "results")
+ ptp.saveResult(self._fc, h5)
+ h5.close()
+
+ km = KnowledgeManager.getInstance()
+
+ km.registerURL('file://'+h5name)
+
+ km_fc = km.getDataContainer(self._fc.id)
+
+ self.assertEqual(self._fc, km_fc)
+
+ os.remove(h5name)
+
+ def testGetHTTPFile(self):
+
+ host = "omnibus.uni-freiburg.de"
+ remote_dir = "/~mr78/pyphant-test"
+ url = "http://"+host+remote_dir+"/knowledgemanager-http-test.h5"
+
+ # Get remote file and load DataContainer
+ filename, headers = urllib.urlretrieve(url)
+ h5 = tables.openFile(filename)
+ for g in h5.walkGroups("/results"):
+ if (len(g._v_attrs.TITLE)>0) \
+ and (r"\Psi" in g._v_attrs.shortname):
+ http_fc = ptp.loadField(h5,g)
+
+ km = KnowledgeManager.getInstance()
+
+ km.registerURL(url)
+
+ km_fc = km.getDataContainer(http_fc.id)
+
+ self.assertEqual(http_fc, km_fc)
+
+ os.remove(filename)
+
+ def testGetDataContainer(self):
+ km = KnowledgeManager.getInstance()
+
+ km.registerDataContainer(self._fc)
+
+ km_fc = km.getDataContainer(self._fc.id)
+
+ self.assertEqual(self._fc, km_fc)
+
+ def testExceptions(self):
+ km = KnowledgeManager.getInstance()
+
+ #invalid id
+
+ #DataContainer not sealed
+
+ #Local file not readable
+
+ #Register empty hdf
+
+if __name__ == "__main__":
+ import sys
+
+ logger = logging.getLogger('pyphant')
+
+
+ hdlr = logging.StreamHandler(sys.stderr)
+ formatter = logging.Formatter('[%(name)s|%(levelname)s] %(message)s')
+ hdlr.setFormatter(formatter)
+
+ logger.addHandler(hdlr)
+ logger.setLevel(logging.DEBUG)
+
+ if len(sys.argv) == 1:
+ unittest.main()
+ else:
+ suite = unittest.TestLoader().loadTestsFromTestCase(eval(sys.argv[1:][0]))
+ unittest.TextTestRunner().run(suite)
Modified: trunk/src/workers/ImageProcessing/ImageProcessing/DiffWorker.py
===================================================================
--- trunk/src/workers/ImageProcessing/ImageProcessing/DiffWorker.py 2009-04-02 13:39:35 UTC (rev 628)
+++ trunk/src/workers/ImageProcessing/ImageProcessing/DiffWorker.py 2009-04-02 16:38:39 UTC (rev 629)
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
-# Copyright (c) 2006-2007, Rectorate of the University of Freiburg
+# Copyright (c) 2006-2009, Rectorate of the University of Freiburg
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
@@ -45,10 +45,13 @@
class DiffWorker(Worker.Worker):
API = 2
- VERSION = 1
+ VERSION = 2
REVISION = "$Revision$"[11:-1]
- name="Diff Worker"
- _params = [("absolute", u"Return absolute of difference: ", [u"Yes", u"No"], None)]
+ name=u"Diff Worker"
+ _params = [("absolute", u"Return absolute of difference: ", [u"Yes", u"No"], None),
+ ("longname",u"Name of result",'default',None),
+ ("symbol",u"Symbol of result",'default',None)]
+
_sockets = [ ("image1", Connectors.TYPE_IMAGE),
("image2", Connectors.TYPE_IMAGE)]
@@ -57,5 +60,9 @@
result = image1 - image2
if self.paramAbsolute.value==u"Yes":
result.data = numpy.abs(result.data)
+ if self.paramLongname.value != 'default':
+ result.longname = self.paramLongname.value
+ if self.paramSymbol.value != 'default':
+ result.shortname = self.paramSymbol.value
result.seal()
return result
Modified: trunk/src/workers/fmfile/fmfile/FMFWriter.py
===================================================================
--- trunk/src/workers/fmfile/fmfile/FMFWriter.py 2009-04-02 13:39:35 UTC (rev 628)
+++ trunk/src/workers/fmfile/fmfile/FMFWriter.py 2009-04-02 16:38:39 UTC (rev 629)
@@ -68,13 +68,25 @@
fc.add_reference_item('author', USER)
if len(fieldContainer.data.shape)==1:
dim = fieldContainer.dimensions[0]
- data = numpy.vstack([dim.data, fieldContainer.data])
+ if fieldContainer.error == None:
+ data = numpy.vstack([dim.data, fieldContainer.data])
+ else:
+ data = numpy.vstack([dim.data, fieldContainer.data,fieldContainer.error])
tab = factory.gen_table(data.transpose())
tab.add_column_def(dim.longname, dim.shortname, str(dim.unit))
+ if fieldContainer.error == None:
+ errorSymbol = None
+ else:
+ errorSymbol = u"\\Delta_{%s}" % fieldContainer.shortname
tab.add_column_def(fieldContainer.longname,
fieldContainer.shortname,
str(fieldContainer.unit),
- dependencies = [dim.shortname])
+ dependencies = [dim.shortname],
+ error = errorSymbol)
+ if fieldContainer.error != None:
+ tab.add_column_def(u"encertainty of %s" % fieldContainer.longname,
+ errorSymbol,
+ str(fieldContainer.unit))
elif fieldContainer.dimensions[0].isIndex():
dim = fieldContainer.dimensions[-1]
try:
Modified: trunk/src/workers/fmfile/fmfile/fmfgen.py
===================================================================
--- trunk/src/workers/fmfile/fmfile/fmfgen.py 2009-04-02 13:39:35 UTC (rev 628)
+++ trunk/src/workers/fmfile/fmfile/fmfgen.py 2009-04-02 16:38:39 UTC (rev 629)
@@ -222,7 +222,7 @@
class _ColumnDef(_Item):
def __init__(self, longname, shortname, unit=None,
- deps=[], format=DEFAULT_COL_FORMAT, *args, **kwds):
+ deps=[], error = None,format=DEFAULT_COL_FORMAT, *args, **kwds):
"""Column definition as item for a data definition section.
longname -- long name for the column, human-readable
@@ -230,6 +230,7 @@
unit -- unit like 'm/s' (optional, default: None)
deps -- list of dependencies, which are themselves short names
of other columns (optional, default: [])
+ error -- Reference to the column symbol listing the respective error
format -- format specifier used for column values (optional)
The instances given for 'longname', 'shortname', 'deps' items and 'unit'
@@ -244,17 +245,21 @@
value += deps[-1]+RPAREN_DEPEND
if unit is not None:
value += u" [%s]" % (unit,)
+ if error is not None:
+ value += u" +- %s" % (error,)
super(_ColumnDef,self).__init__( longname, value, *args, **kwds)
self._shortname = shortname
self._unit = unit
self._deps = deps
+ self._error = error
self._format = format
longname = _AutoProperty('_metatag')
shortname = _AutoProperty('_shortname')
unit = _AutoProperty('_unit')
deps = _AutoProperty('_deps')
+ error = _AutoProperty('_error')
format = _AutoProperty('_format')
class _Section(_FMFElement):
@@ -400,6 +405,7 @@
% (self._shortname,))
def add_column_def(self, longname, shortname, unit=None, dependencies=[],
+ error = None,
format=DEFAULT_COL_FORMAT, arg_coding=None): # , data=None):
"""Add a column definition to the table.
@@ -417,7 +423,7 @@
# data -- sequence with references to data items; length must be
# equal to current number of rows in the table; data
# is joined to the current table (optional)
- coldef = self._factory._gen_column_def(longname, shortname, unit, dependencies,
+ coldef = self._factory._gen_column_def(longname, shortname, unit, dependencies,error,
format, arg_coding)
self._coldefs.append(coldef)
#if not data is None:
@@ -635,7 +641,7 @@
kwds.update(self._element_kwds) # mix in the coding and line marker information
return _Item(metatag, value, *args, **kwds)
- def _gen_column_def(self, longname, shortname, unit=None, dependencies=[],
+ def _gen_column_def(self, longname, shortname, unit=None, dependencies=[], error = None,
format=DEFAULT_COL_FORMAT, arg_coding=None, *args, **kwds):
"""Generate a column definition.
@@ -664,7 +670,7 @@
for i,d in enumerate(dependencies):
dependencies[i] = assure_unicode(d, arg_coding)
kwds.update(self._element_kwds) # mix in the coding and line marker information
- return _ColumnDef(longname, shortname, unit, dependencies, format, *args, **kwds)
+ return _ColumnDef(longname, shortname, unit, dependencies, error, format, *args, **kwds)
def gen_section(self, tag, arg_coding=None, *args, **kwds):
"""Generate a section for latter inclusion in a FMF file.
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|