Menu
▾
▴
SF.net SVN: matplotlib: [4449] trunk/toolkits/basemap
|
From: <js...@us...> - 2007-11-26 18:30:20
|
Revision: 4449
http://matplotlib.svn.sourceforge.net/matplotlib/?rev=4449&view=rev
Author: jswhit
Date: 2007-11-26 10:29:14 -0800 (Mon, 26 Nov 2007)
Log Message:
-----------
add the ability to read remote datasets over http using the opendap
client by Roberto D'Almeida (included). NetCDFFile modified so
that it uses the opendap client if the filename begins with 'http'.
Modified Paths:
--------------
trunk/toolkits/basemap/Changelog
trunk/toolkits/basemap/MANIFEST.in
trunk/toolkits/basemap/examples/fcstmaps.py
trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py
trunk/toolkits/basemap/setup.py
Added Paths:
-----------
trunk/toolkits/basemap/lib/dap/
trunk/toolkits/basemap/lib/dap/__init__.py
trunk/toolkits/basemap/lib/dap/client.py
trunk/toolkits/basemap/lib/dap/dtypes.py
trunk/toolkits/basemap/lib/dap/exceptions.py
trunk/toolkits/basemap/lib/dap/helper.py
trunk/toolkits/basemap/lib/dap/lib.py
trunk/toolkits/basemap/lib/dap/parsers/
trunk/toolkits/basemap/lib/dap/parsers/__init__.py
trunk/toolkits/basemap/lib/dap/parsers/das.py
trunk/toolkits/basemap/lib/dap/parsers/dds.py
trunk/toolkits/basemap/lib/dap/util/
trunk/toolkits/basemap/lib/dap/util/__init__.py
trunk/toolkits/basemap/lib/dap/util/filter.py
trunk/toolkits/basemap/lib/dap/util/http.py
trunk/toolkits/basemap/lib/dap/util/ordereddict.py
trunk/toolkits/basemap/lib/dap/util/safeeval.py
trunk/toolkits/basemap/lib/dap/util/wsgi_intercept.py
trunk/toolkits/basemap/lib/dap/xdr.py
Modified: trunk/toolkits/basemap/Changelog
===================================================================
--- trunk/toolkits/basemap/Changelog 2007-11-26 17:23:18 UTC (rev 4448)
+++ trunk/toolkits/basemap/Changelog 2007-11-26 18:29:14 UTC (rev 4449)
@@ -1,3 +1,5 @@
+ * modify NetCDFFile to use dap module to read remote
+ datasets over http. Include dap module.
* modify NetCDFFile to automatically apply scale_factor
and add_offset, and return masked arrays masked where
data == missing_value or _FillValue.
Modified: trunk/toolkits/basemap/MANIFEST.in
===================================================================
--- trunk/toolkits/basemap/MANIFEST.in 2007-11-26 17:23:18 UTC (rev 4448)
+++ trunk/toolkits/basemap/MANIFEST.in 2007-11-26 18:29:14 UTC (rev 4449)
@@ -74,6 +74,9 @@
include pyshapelib/shapelib/*.c pyshapelib/shapelib/*.h
include MANIFEST.in
recursive-include geos-2.2.3 *
+recursive-include lib/dap *
+recursive-include lib/dbflib *
+recursive-include lib/shapelib *
include lib/matplotlib/toolkits/basemap/data/5minmask.bin
include lib/matplotlib/toolkits/basemap/data/GL27
include lib/matplotlib/toolkits/basemap/data/countries_c.dat
Modified: trunk/toolkits/basemap/examples/fcstmaps.py
===================================================================
--- trunk/toolkits/basemap/examples/fcstmaps.py 2007-11-26 17:23:18 UTC (rev 4448)
+++ trunk/toolkits/basemap/examples/fcstmaps.py 2007-11-26 18:29:14 UTC (rev 4449)
@@ -1,19 +1,14 @@
# this example reads today's numerical weather forecasts
# from the NOAA OpenDAP servers and makes a multi-panel plot.
-# Requires the pyDAP module (a pure-python module)
-# from http://pydap.org, and an active intenet connection.
-
-try:
- from dap import client
-except:
- raise ImportError,"requires pyDAP module (version 2.1 or higher) from http://pydap.org"
-from pylab import title, show, figure, cm, arange, frange, figtext, \
- meshgrid, axes, colorbar, where, amin, amax, around
+from pylab import title, show, figure, cm, figtext, \
+ meshgrid, axes, colorbar
+import numpy
import sys
from matplotlib.numerix import ma
import datetime
-from matplotlib.toolkits.basemap import Basemap
+from matplotlib.toolkits.basemap import Basemap, NetCDFFile, addcyclic
+
hrsgregstart = 13865688 # hrs from 00010101 to 15821015 in Julian calendar.
# times in many datasets use mixed Gregorian/Julian calendar, datetime
# module uses a proleptic Gregorian calendar. So, I use datetime to compute
@@ -48,12 +43,11 @@
YYYYMM = YYYYMMDD[0:6]
# set OpenDAP server URL.
-HH='09'
-URLbase="http://nomad3.ncep.noaa.gov:9090/dods/sref/sref"
-URL=URLbase+YYYYMMDD+"/sref_eta_ctl1_"+HH+"z"
+URLbase="http://nomad3.ncep.noaa.gov:9090/dods/mrf/mrf"
+URL=URLbase+YYYYMMDD+'/mrf'+YYYYMMDD
print URL+'\n'
try:
- data = client.open(URL)
+ data = NetCDFFile(URL)
except:
msg = """
opendap server not providing the requested data.
@@ -61,14 +55,14 @@
raise IOError, msg
-# read levels, lats,lons,times.
+# read lats,lons,times.
-print data.keys()
-levels = data['lev']
-latitudes = data['lat']
-longitudes = data['lon']
-fcsttimes = data['time']
-times = fcsttimes[:]
+print data.variables.keys()
+latitudes = data.variables['lat']
+longitudes = data.variables['lon']
+fcsttimes = data.variables['time']
+times = fcsttimes[0:6] # first 6 forecast times.
+ntimes = len(times)
# put forecast times in YYYYMMDDHH format.
verifdates = []
fcsthrs=[]
@@ -79,55 +73,43 @@
verifdates.append(fdate.strftime('%Y%m%d%H'))
print fcsthrs
print verifdates
-levs = levels[:]
lats = latitudes[:]
-lons = longitudes[:]
-lons, lats = meshgrid(lons,lats)
+nlats = len(lats)
+lons1 = longitudes[:]
+nlons = len(lons1)
# unpack 2-meter temp forecast data.
-t2mvar = data['tmp2m']
-missval = t2mvar.missing_value
-t2m = t2mvar[:,:,:]
-if missval < 0:
- t2m = ma.masked_values(where(t2m>-1.e20,t2m,1.e20), 1.e20)
-else:
- t2m = ma.masked_values(where(t2m<1.e20,t2m,1.e20), 1.e20)
-t2min = amin(t2m.compressed()); t2max= amax(t2m.compressed())
-print t2min,t2max
-clevs = frange(around(t2min/10.)*10.-5.,around(t2max/10.)*10.+5.,4)
-print clevs[0],clevs[-1]
-llcrnrlat = 22.0
-urcrnrlat = 48.0
-latminout = 22.0
-llcrnrlon = -125.0
-urcrnrlon = -60.0
-standardpar = 50.0
-centerlon=-105.
-# create Basemap instance for Lambert Conformal Conic projection.
-m = Basemap(llcrnrlon=llcrnrlon,llcrnrlat=llcrnrlat,
- urcrnrlon=urcrnrlon,urcrnrlat=urcrnrlat,
+t2mvar = data.variables['tmp2m']
+t2min = t2mvar[0:ntimes,:,:]
+t2m = numpy.zeros((ntimes,nlats,nlons+1),t2min.dtype)
+# create Basemap instance for Orthographic projection.
+m = Basemap(lon_0=-105,lat_0=40,
rsphere=6371200.,
- resolution='l',area_thresh=5000.,projection='lcc',
- lat_1=standardpar,lon_0=centerlon)
+ resolution='c',area_thresh=5000.,projection='ortho')
+# add wrap-around point in longitude.
+for nt in range(ntimes):
+ t2m[nt,:,:], lons = addcyclic(t2min[nt,:,:], lons1)
+# convert to celsius.
+t2m = t2m-273.15
+# contour levels
+clevs = numpy.arange(-30,30.1,2.)
+lons, lats = meshgrid(lons, lats)
x, y = m(lons, lats)
# create figure.
-fig=figure(figsize=(8,8))
-yoffset = (m.urcrnry-m.llcrnry)/30.
-for npanel,fcsthr in enumerate(arange(0,72,12)):
- nt = fcsthrs.index(fcsthr)
- ax = fig.add_subplot(320+npanel+1)
- #cs = m.contour(x,y,t2m[nt,:,:],clevs,colors='k')
- cs = m.contourf(x,y,t2m[nt,:,:],clevs,cmap=cm.jet)
- m.drawcoastlines()
- m.drawstates()
+fig=figure(figsize=(6,8))
+# make subplots.
+for nt,fcsthr in enumerate(fcsthrs):
+ ax = fig.add_subplot(321+nt)
+ cs = m.contourf(x,y,t2m[nt,:,:],clevs,cmap=cm.jet,extend='both')
+ m.drawcoastlines(linewidth=0.5)
m.drawcountries()
- m.drawparallels(arange(25,75,20),labels=[1,0,0,0],fontsize=8,fontstyle='oblique')
- m.drawmeridians(arange(-140,0,20),labels=[0,0,0,1],fontsize=8,yoffset=yoffset,fontstyle='oblique')
+ m.drawparallels(numpy.arange(-80,81,20))
+ m.drawmeridians(numpy.arange(0,360,20))
# panel title
- title(repr(fcsthr)+'-h forecast valid '+verifdates[nt],fontsize=12)
+ title(repr(fcsthr)+'-h forecast valid '+verifdates[nt],fontsize=9)
# figure title
-figtext(0.5,0.95,u"2-m temp (\N{DEGREE SIGN}K) forecasts from %s"%verifdates[0],
+figtext(0.5,0.95,u"2-m temp (\N{DEGREE SIGN}C) forecasts from %s"%verifdates[0],
horizontalalignment='center',fontsize=14)
# a single colorbar.
cax = axes([0.1, 0.03, 0.8, 0.025])
Added: trunk/toolkits/basemap/lib/dap/__init__.py
===================================================================
--- trunk/toolkits/basemap/lib/dap/__init__.py (rev 0)
+++ trunk/toolkits/basemap/lib/dap/__init__.py 2007-11-26 18:29:14 UTC (rev 4449)
@@ -0,0 +1,12 @@
+"""A Python implementation of the Data Access Protocol (DAP).
+
+Pydap is a Python module implementing the Data Access Protocol (DAP)
+written from scratch. The module implements a DAP client, allowing
+transparent and efficient access to dataset stored in DAP server, and
+also implements a DAP server able to serve data from a variety of
+formats.
+
+For more information about the protocol, please check http://opendap.org.
+"""
+
+__import__('pkg_resources').declare_namespace(__name__)
Added: trunk/toolkits/basemap/lib/dap/client.py
===================================================================
--- trunk/toolkits/basemap/lib/dap/client.py (rev 0)
+++ trunk/toolkits/basemap/lib/dap/client.py 2007-11-26 18:29:14 UTC (rev 4449)
@@ -0,0 +1,67 @@
+__author__ = "Roberto De Almeida <ro...@py...>"
+
+import dap.lib
+from dap.util.http import openurl
+from dap.exceptions import ClientError
+
+
+def open(url, cache=None, username=None, password=None, verbose=False):
+ """Connect to a remote dataset.
+
+ This function opens a dataset stored in a DAP server:
+
+ >>> dataset = open(url, cache=None, username=None, password=None, verbose=False):
+
+ You can specify a cache location (a directory), so that repeated
+ accesses to the same URL avoid the network.
+
+ The username and password may be necessary if the DAP server requires
+ authentication. The 'verbose' option will make pydap print all the
+ URLs that are acessed.
+ """
+ # Set variables on module namespace.
+ dap.lib.VERBOSE = verbose
+
+ if url.startswith('http'):
+ for response in [_ddx, _ddsdas]:
+ dataset = response(url, cache, username, password)
+ if dataset: return dataset
+ else:
+ raise ClientError("Unable to open dataset.")
+ else:
+ from dap.plugins.lib import loadhandler
+ from dap.helper import walk
+
+ # Open a local file. This is a clever hack. :)
+ handler = loadhandler(url)
+ dataset = handler._parseconstraints()
+
+ # Unwrap any arrayterators in the dataset.
+ for var in walk(dataset):
+ try: var.data = var.data._var
+ except: pass
+
+ return dataset
+
+
+def _ddsdas(baseurl, cache, username, password):
+ ddsurl, dasurl = '%s.dds' % baseurl, '%s.das' % baseurl
+
+ # Get metadata.
+ respdds, dds = openurl(ddsurl, cache, username, password)
+ respdas, das = openurl(dasurl, cache, username, password)
+
+ if respdds['status'] == '200' and respdas['status'] == '200':
+ from dap.parsers.dds import DDSParser
+ from dap.parsers.das import DASParser
+
+ # Build dataset.
+ dataset = DDSParser(dds, ddsurl, cache, username, password).parse()
+
+ # Add attributes.
+ dataset = DASParser(das, dasurl, dataset).parse()
+ return dataset
+
+
+def _ddx(baseurl, cache, username, password):
+ pass
Added: trunk/toolkits/basemap/lib/dap/dtypes.py
===================================================================
--- trunk/toolkits/basemap/lib/dap/dtypes.py (rev 0)
+++ trunk/toolkits/basemap/lib/dap/dtypes.py 2007-11-26 18:29:14 UTC (rev 4449)
@@ -0,0 +1,529 @@
+"""DAP variables.
+
+This module is a Python implementation of the DAP data model.
+"""
+
+__author__ = "Roberto De Almeida <ro...@py...>"
+
+import copy
+import itertools
+
+from dap.lib import quote, to_list, _quote
+from dap.util.ordereddict import odict
+from dap.util.filter import get_filters
+
+__all__ = ['StructureType', 'SequenceType', 'DatasetType', 'GridType', 'ArrayType', 'BaseType',
+ 'Float', 'Float0', 'Float8', 'Float16', 'Float32', 'Float64', 'Int', 'Int0', 'Int8',
+ 'Int16', 'Int32', 'Int64', 'UInt16', 'UInt32', 'UInt64', 'Byte', 'String', 'Url']
+
+_basetypes = ['Float32', 'Float64', 'Int16', 'Int32', 'UInt16', 'UInt32', 'Byte', 'String', 'Url']
+_constructors = ['StructureType', 'SequenceType', 'DatasetType', 'GridType', 'ArrayType']
+
+# Constants.
+Float = 'Float64'
+Float0 = 'Float64'
+Float8 = 'Float32'
+Float16 = 'Float32'
+Float32 = 'Float32'
+Float64 = 'Float64'
+Int = 'Int32'
+Int0 = 'Int32'
+Int8 = 'Byte'
+Int16 = 'Int16'
+Int32 = 'Int32'
+Int64 = 'Int32'
+UInt16 = 'UInt16'
+UInt32 = 'UInt32'
+UInt64 = 'UInt32'
+UInt8 = 'Byte'
+Byte = 'Byte'
+String = 'String'
+Url = 'Url'
+
+typemap = {
+ # numpy
+ 'd': Float64,
+ 'f': Float32,
+ 'l': Int32,
+ 'b': Byte,
+ 'h': Int16,
+ 'q': Int32,
+ 'H': UInt16,
+ 'L': UInt32,
+ 'Q': UInt32,
+ 'B': Byte,
+ 'S': String,
+ }
+
+
+class StructureType(odict):
+ """Structure contructor.
+
+ A structure is a dict-like object, which can hold other DAP variables.
+ Structures have a 'data' attribute that combines the data from the
+ stored variables when read, and propagates the data to the variables
+ when set.
+
+ This behaviour can be bypassed by setting the '_data' attribute; in
+ this case, no data is propagated, and further reads do not combine the
+ data from the stored variables.
+ """
+
+ def __init__(self, name='', attributes=None):
+ odict.__init__(self)
+ self.name = quote(name)
+ self.attributes = attributes or {}
+
+ self._id = name
+ self._filters = []
+ self._data = None
+
+ def __iter__(self):
+ # Iterate over the variables contained in the structure.
+ return self.itervalues()
+
+ walk = __iter__
+
+ def __getattr__(self, attr):
+ # Try to return stored variable.
+ try:
+ return self[attr]
+ except KeyError:
+ # Try to return attribute from self.attributes.
+ try: return self.attributes[attr]
+ except KeyError: raise AttributeError
+
+ def __setitem__(self, key, item):
+ # Assign a new variable and apply the proper id.
+ self._dict.__setitem__(key, item)
+ if key not in self._keys: self._keys.append(key)
+
+ # Ensure that stored objects have the proper id.
+ item._set_id(self._id)
+
+ def _get_data(self):
+ if self._data is not None:
+ return self._data
+ else:
+ return [var.data for var in self.values()]
+
+ def _set_data(self, data):
+ # Propagate the data to the stored variables.
+ for data_, var in itertools.izip(data, self.values()):
+ var.data = data_
+
+ data = property(_get_data, _set_data)
+
+ def _get_id(self):
+ return self._id
+
+ def _set_id(self, parent=None):
+ if parent:
+ self._id = '%s.%s' % (parent, self.name)
+ else:
+ self._id = self.name
+
+ # Propagate id to stored variables.
+ for var in self.values(): var._set_id(self._id)
+
+ id = property(_get_id) # Read-only.
+
+ def _get_filters(self):
+ return self._filters
+
+ def _set_filters(self, f):
+ self._filters.append(f)
+
+ # Propagate filter to stored variables.
+ for var in self.values(): var._set_filters(f)
+
+ filters = property(_get_filters, _set_filters)
+
+ def __copy__(self):
+ out = self.__class__(name=self.name, attributes=self.attributes.copy())
+ out._id = self._id
+ out._filters = self._filters[:]
+ out._data = self._data
+
+ # Stored variables *are not* copied.
+ for k, v in self.items():
+ out[k] = v
+ return out
+
+ def __deepcopy__(self, memo=None, _nil=[]):
+ out = self.__class__(name=self.name, attributes=self.attributes.copy())
+ out._id = self._id
+ out._filters = self._filters[:]
+ out._data = self._data
+
+ # Stored variables *are* (deep) copied.
+ for k, v in self.items():
+ out[k] = copy.deepcopy(v)
+ return out
+
+
+class DatasetType(StructureType):
+ """Dataset constructor.
+
+ A dataset is very similar to a structure -- the main difference is that
+ its name is not used when composing the fully qualified name of stored
+ variables.
+ """
+
+ def __setitem__(self, key, item):
+ self._dict.__setitem__(key, item)
+ if key not in self._keys: self._keys.append(key)
+
+ # Set the id. Here the parent should be None, since the dataset
+ # id is not part of the fully qualified name.
+ item._set_id(None)
+
+ def _set_id(self, parent=None):
+ self._id = self.name
+
+ # Propagate id.
+ for var in self.values(): var._set_id(None)
+
+
+class SequenceType(StructureType):
+ """Sequence constructor.
+
+ A sequence contains ordered data, corresponding to the records in
+ a sequence of structures with the same stored variables.
+ """
+ # Nesting level. Sequences inside sequences have a level 2, and so on.
+ level = 1
+
+ def __setitem__(self, key, item):
+ # Assign a new variable and apply the proper id.
+ self._dict.__setitem__(key, item)
+ if key not in self._keys: self._keys.append(key)
+
+ # Ensure that stored objects have the proper id.
+ item._set_id(self._id)
+
+ # If the variable is a sequence, set the nesting level.
+ def set_level(seq, level):
+ if isinstance(seq, SequenceType):
+ seq.level = level
+ for child in seq.walk(): set_level(child, level+1)
+ set_level(item, self.level+1)
+
+ def walk(self):
+ # Walk over the variables contained in the structure.
+ return self.itervalues()
+
+ def _get_data(self):
+ # This is similar to the structure _get_data method, except that data
+ # is combined from stored variables using zip(), i.e., grouped values
+ # from each variable.
+ if self._data is not None:
+ return self._data
+ else:
+ return _build_data(self.level, *[var.data for var in self.values()])
+
+ def _set_data(self, data):
+ for data_, var in itertools.izip(_propagate_data(self.level, data), self.values()):
+ var.data = data_
+
+ data = property(_get_data, _set_data)
+
+ def __iter__(self):
+ """
+ When iterating over a sequence, we yield structures containing the
+ corresponding data (first record, second, etc.).
+ """
+ out = self.__deepcopy__()
+
+ # Set server-side filters. When the sequence is iterated in a
+ # listcomp/genexp, this function inspects the stack and tries to
+ # build a server-side filter from the client-side filter. This
+ # is voodoo black magic, take care.
+ filters = get_filters(out)
+ for filter_ in filters:
+ out._set_filters(filter_)
+
+ for values in out.data:
+ # Yield a nice structure.
+ struct_ = StructureType(name=out.name, attributes=out.attributes)
+ for data, name in zip(values, out.keys()):
+ var = struct_[name] = out[name].__deepcopy__()
+ var.data = data
+ # Set the id. This is necessary since the new structure is not
+ # contained inside a dataset.
+ parent = out._id[:-len(out.name)-1]
+ struct_._set_id(parent)
+ yield struct_
+
+ def filter(self, *filters):
+ # Make a copy of the sequence.
+ out = self.__deepcopy__()
+
+ # And filter it according to the selection expression.
+ for filter_ in filters:
+ out._set_filters(_quote(filter_))
+ return out
+
+
+class GridType(object):
+ """Grid constructor.
+
+ A grid is a constructor holding an 'array' variable. The array has its
+ dimensions mapped to 'maps' stored in the grid (lat, lon, time, etc.).
+ Most of the requests are simply passed onto the stored array.
+ """
+
+ def __init__(self, name='', array=None, maps=None, attributes=None):
+ self.name = quote(name)
+ self.array = array
+ self.maps = maps or odict()
+ self.attributes = attributes or {}
+
+ self._id = name
+ self._filters = []
+
+ def __len__(self):
+ return self.array.shape[0]
+
+ def __iter__(self):
+ # Iterate over the grid. Yield the array and then the maps.
+ yield self.array
+ for map_ in self.maps.values(): yield map_
+
+ walk = __iter__
+
+ def __getattr__(self, attr):
+ # Try to return attribute from self.attributes.
+ try:
+ return self.attributes[attr]
+ except KeyError:
+ raise AttributeError
+
+ def __getitem__(self, index):
+ # Return data from the array.
+ return self.array[index]
+
+ def _get_data(self):
+ return self.array.data
+
+ def _set_data(self, data):
+ self.array.data = data
+
+ data = property(_get_data, _set_data)
+
+ def _get_id(self):
+ return self._id
+
+ def _set_id(self, parent=None):
+ if parent: self._id = '%s.%s' % (parent, self.name)
+ else: self._id = self.name
+
+ # Propagate id to array and maps.
+ if self.array: self.array._set_id(self._id)
+ for map_ in self.maps.values():
+ map_._set_id(self._id)
+
+ id = property(_get_id)
+
+ def _get_filters(self):
+ return self._filters
+
+ def _set_filters(self, f):
+ self.filters.append(f)
+
+ # Propagate filter.
+ self.array._set_filters(f)
+ for map_ in self.maps.values():
+ map_._set_filters(f)
+
+ filters = property(_get_filters, _set_filters)
+
+ def _get_dimensions(self):
+ # Return dimensions from stored maps.
+ return tuple(self.maps.keys())
+
+ dimensions = property(_get_dimensions)
+
+ def _get_shape(self):
+ return self.array.shape
+
+ def _set_shape(self, shape):
+ self.array.shape = shape
+
+ shape = property(_get_shape, _set_shape)
+
+ def _get_type(self):
+ return self.array.type
+
+ def _set_type(self, type):
+ self.array.type = type
+
+ type = property(_get_type, _set_type)
+
+ def __copy__(self):
+ out = self.__class__(name=self.name, array=self.array, maps=self.maps, attributes=self.attributes.copy())
+ out._id = self._id
+ out._filters = self._filters[:]
+ return out
+
+ def __deepcopy__(self, memo=None, _nil=[]):
+ out = self.__class__(name=self.name, attributes=self.attributes.copy())
+ out.array = copy.deepcopy(self.array)
+ out.maps = copy.deepcopy(self.maps)
+ out._id = self._id
+ out._filters = self._filters[:]
+ return out
+
+
+class BaseType(object):
+ """DAP Base type.
+
+ Variable holding a single value, or an iterable if it's stored inside
+ a sequence. It's the fundamental DAP variable, which actually holds
+ data (together with arrays).
+ """
+
+ def __init__(self, name='', data=None, type=None, attributes=None):
+ self.name = quote(name)
+ self.data = data
+ self.attributes = attributes or {}
+
+ if type in _basetypes: self.type = type
+ else: self.type = typemap.get(type, Int32)
+
+ self._id = name
+ self._filters = []
+
+ def __iter__(self):
+ # Yield the stored value.
+ # Perhaps we should raise StopIteration?
+ yield self.data
+
+ def __getattr__(self, attr):
+ # Try to return attribute from self.attributes.
+ try:
+ return self.attributes[attr]
+ except KeyError:
+ raise AttributeError
+
+ def __getitem__(self, key):
+ # Return data from the array.
+ return self.data[key]
+
+ def __setitem__(self, key, item):
+ # Assign a new variable and apply the proper id.
+ self.data.__setitem__(key, item)
+
+ def _get_id(self):
+ return self._id
+
+ def _set_id(self, parent=None):
+ if parent: self._id = '%s.%s' % (parent, self.name)
+ else: self._id = self.name
+
+ id = property(_get_id) # Read-only.
+
+ def _get_filters(self):
+ return self._filters
+
+ def _set_filters(self, f):
+ self._filters.append(f)
+
+ # Propagate to data, if it's a Proxy object.
+ if hasattr(self.data, 'filters'):
+ self.data.filters = self._filters
+
+ filters = property(_get_filters, _set_filters)
+
+ def __copy__(self):
+ out = self.__class__(name=self.name, data=self.data, type=self.type, attributes=self.attributes.copy())
+ out._id = self._id
+ out._filters = self._filters[:]
+ return out
+
+ def __deepcopy__(self, memo=None, _nil=[]):
+ out = self.__class__(name=self.name, type=self.type, attributes=self.attributes.copy())
+
+ try:
+ out.data = copy.copy(self.data)
+ except TypeError:
+ self.data = to_list(self.data)
+ out.data = copy.copy(self.data)
+
+ out._id = self._id
+ out._filters = self._filters[:]
+ return out
+
+ # This allows the variable to be compared to numbers.
+ def __ge__(self, other): return self.data >= other
+ def __gt__(self, other): return self.data > other
+ def __le__(self, other): return self.data <= other
+ def __lt__(self, other): return self.data < other
+ def __eq__(self, other): return self.data == other
+
+
+class ArrayType(BaseType):
+ """An array of BaseType variables.
+
+ Although the DAP supports arrays of any DAP variables, pydap can only
+ handle arrays of base types. This makes the ArrayType class very
+ similar to a BaseType, with the difference that it'll hold an array
+ of data in its 'data' attribute.
+
+ Array of constructors will not be supported until Python has a
+ native multi-dimensional array type.
+ """
+
+ def __init__(self, name='', data=None, shape=None, dimensions=None, type=None, attributes=None):
+ self.name = quote(name)
+ self.data = data
+ self.shape = shape or ()
+ self.dimensions = dimensions or ()
+ self.attributes = attributes or {}
+
+ if type in _basetypes: self.type = type
+ else: self.type = typemap.get(type, Int32)
+
+ self._id = name
+ self._filters = []
+
+ def __len__(self):
+ return self.shape[0]
+
+ def __copy__(self):
+ out = self.__class__(name=self.name, data=self.data, shape=self.shape, dimensions=self.dimensions, type=self.type, attributes=self.attributes.copy())
+ out._id = self._id
+ out._filters = self._filters[:]
+ return out
+
+ def __deepcopy__(self, memo=None, _nil=[]):
+ out = self.__class__(name=self.name, shape=self.shape, dimensions=self.dimensions, type=self.type, attributes=self.attributes.copy())
+
+ try:
+ out.data = copy.copy(self.data)
+ except TypeError:
+ self.data = to_list(self.data)
+ out.data = copy.copy(self.data)
+
+ out._id = self._id
+ out._filters = self._filters[:]
+ return out
+
+
+# Functions for propagating data up and down in sequences.
+# I'm not 100% sure how this works.
+def _build_data(level, *vars_):
+ if level > 0:
+ out = [_build_data(level-1, *els) for els in itertools.izip(*vars_)]
+ else:
+ out = vars_
+
+ return out
+
+def _propagate_data(level, vars_):
+ if level > 0:
+ out = zip(*[_propagate_data(level-1, els) for els in vars_])
+ else:
+ out = vars_
+
+ return out
Added: trunk/toolkits/basemap/lib/dap/exceptions.py
===================================================================
--- trunk/toolkits/basemap/lib/dap/exceptions.py (rev 0)
+++ trunk/toolkits/basemap/lib/dap/exceptions.py 2007-11-26 18:29:14 UTC (rev 4449)
@@ -0,0 +1,49 @@
+"""DAP exceptions.
+
+These exceptions are mostly used by the server. When an exception is
+captured, a proper error message is displayed (according to the DAP
+2.0 spec), with information about the exception and the error code
+associated with it.
+
+The error codes are attributed using the "first come, first serve"
+algorithm.
+"""
+
+__author__ = "Roberto De Almeida <ro...@py...>"
+
+
+class DapError(Exception):
+ """Base DAP exception."""
+ def __init__(self, value):
+ self.value = value
+
+ def __str__(self):
+ return repr(self.value)
+
+
+class ClientError(DapError):
+ """Generic error with the client."""
+ code = 100
+
+
+class ServerError(DapError):
+ """Generic error with the server."""
+ code = 200
+
+class ConstraintExpressionError(ServerError):
+ """Exception raised when an invalid constraint expression is given."""
+ code = 201
+
+
+class PluginError(DapError):
+ """Generic error with a plugin."""
+ code = 300
+
+class ExtensionNotSupportedError(PluginError):
+ """Exception raised when trying to open a file not supported by any plugins."""
+ code = 301
+
+class OpenFileError(PluginError):
+ """Exception raised when unable to open a file."""
+ code = 302
+
Added: trunk/toolkits/basemap/lib/dap/helper.py
===================================================================
--- trunk/toolkits/basemap/lib/dap/helper.py (rev 0)
+++ trunk/toolkits/basemap/lib/dap/helper.py 2007-11-26 18:29:14 UTC (rev 4449)
@@ -0,0 +1,803 @@
+"""Helper functions.
+
+These are generic functions used mostly for writing plugins.
+"""
+
+__author__ = "Roberto De Almeida <ro...@py...>"
+
+import sys
+import re
+import operator
+import itertools
+import copy
+from urllib import quote, unquote
+
+from dap.dtypes import *
+from dap.dtypes import _basetypes
+from dap.exceptions import ConstraintExpressionError
+from dap.lib import isiterable
+from dap.util.safeeval import expr_eval
+from dap.util.ordereddict import odict
+
+
+def constrain(dataset, constraints):
+ """A simple example. We create a dataset holding three variables:
+
+ >>> dataset = DatasetType(name='foo')
+ >>> dataset['a'] = BaseType(name='a', type='Byte')
+ >>> dataset['b'] = BaseType(name='b', type='Byte')
+ >>> dataset['c'] = BaseType(name='c', type='Byte')
+
+ Now we give it a CE requesting only the variables ``a`` and ``b``:
+
+ >>> dataset2 = constrain(dataset, 'a,b')
+ >>> print dataset2 #doctest: +ELLIPSIS
+ {'a': <dap.dtypes.BaseType object at ...>, 'b': <dap.dtypes.BaseType object at ...>}
+
+ We can also request the variables in a different order:
+
+ >>> dataset2 = constrain(dataset, 'b,a')
+ >>> print dataset2 #doctest: +ELLIPSIS
+ {'b': <dap.dtypes.BaseType object at ...>, 'a': <dap.dtypes.BaseType object at ...>}
+
+ Another example. A dataset with two structures ``a`` and ``b``:
+
+ >>> dataset = DatasetType(name='foo')
+ >>> dataset['a'] = StructureType(name='a')
+ >>> dataset['a']['a1'] = BaseType(name='a1', type='Byte')
+ >>> dataset['b'] = StructureType(name='b')
+ >>> dataset['b']['b1'] = BaseType(name='b1', type='Byte')
+ >>> dataset['b']['b2'] = BaseType(name='b2', type='Byte')
+
+ If we request the structure ``b`` we should get it complete:
+
+ >>> dataset2 = constrain(dataset, 'a.a1,b')
+ >>> print dataset2 #doctest: +ELLIPSIS
+ {'a': {'a1': <dap.dtypes.BaseType object at ...>}, 'b': {'b1': <dap.dtypes.BaseType object at ...>, 'b2': <dap.dtypes.BaseType object at ...>}}
+
+ >>> dataset2 = constrain(dataset, 'b.b1')
+ >>> print dataset2 #doctest: +ELLIPSIS
+ {'b': {'b1': <dap.dtypes.BaseType object at ...>}}
+
+ Arrays can be sliced. Here we have a ``(2,3)`` array:
+
+ >>> dataset = DatasetType(name='foo')
+ >>> from numpy import array
+ >>> data = array([1,2,3,4,5,6])
+ >>> data.shape = (2,3)
+ >>> dataset['array'] = ArrayType(data=data, name='array', shape=(2,3), type='Int32')
+ >>> dataset2 = constrain(dataset, 'array')
+ >>> from dap.server import SimpleHandler
+ >>> headers, output = SimpleHandler(dataset).dds()
+ >>> print ''.join(output)
+ Dataset {
+ Int32 array[2][3];
+ } foo;
+ <BLANKLINE>
+ >>> print dataset2['array'].data
+ [[1 2 3]
+ [4 5 6]]
+
+ But we request only part of it:
+
+ >>> dataset2 = constrain(dataset, 'array[0:1:1][0:1:1]')
+ >>> headers, output = SimpleHandler(dataset2).dds()
+ >>> print ''.join(output)
+ Dataset {
+ Int32 array[2][2];
+ } foo;
+ <BLANKLINE>
+ >>> print dataset2['array'].data
+ [[1 2]
+ [4 5]]
+
+ The same is valid for grids:
+
+ >>> dataset['grid'] = GridType(name='grid')
+ >>> data = array([1,2,3,4,5,6])
+ >>> data.shape = (2,3)
+ >>> dataset['grid'].array = ArrayType(name='grid', data=data, shape=(2,3), dimensions=('x', 'y'))
+ >>> dataset['grid'].maps['x'] = ArrayType(name='x', data=array([1,2]), shape=(2,))
+ >>> dataset['grid'].maps['y'] = ArrayType(name='y', data=array([1,2,3]), shape=(3,))
+ >>> dataset._set_id()
+ >>> headers, output = SimpleHandler(dataset).dds()
+ >>> print ''.join(output)
+ Dataset {
+ Int32 array[2][3];
+ Grid {
+ Array:
+ Int32 grid[x = 2][y = 3];
+ Maps:
+ Int32 x[x = 2];
+ Int32 y[y = 3];
+ } grid;
+ } foo;
+ <BLANKLINE>
+ >>> dataset2 = constrain(dataset, 'grid[0:1:0][0:1:0]')
+ >>> headers, output = SimpleHandler(dataset2).dds()
+ >>> print ''.join(output)
+ Dataset {
+ Grid {
+ Array:
+ Int32 grid[x = 1][y = 1];
+ Maps:
+ Int32 x[x = 1];
+ Int32 y[y = 1];
+ } grid;
+ } foo;
+ <BLANKLINE>
+ >>> headers, output = SimpleHandler(dataset2).ascii()
+ >>> print ''.join(output)
+ Dataset {
+ Grid {
+ Array:
+ Int32 grid[x = 1][y = 1];
+ Maps:
+ Int32 x[x = 1];
+ Int32 y[y = 1];
+ } grid;
+ } foo;
+ ---------------------------------------------
+ grid.grid
+ [0][0] 1
+ <BLANKLINE>
+ grid.x
+ [0] 1
+ <BLANKLINE>
+ grid.y
+ [0] 1
+ <BLANKLINE>
+ <BLANKLINE>
+ <BLANKLINE>
+ <BLANKLINE>
+
+ Selecting a map from a Grid should return a structure:
+
+ >>> dataset3 = constrain(dataset, 'grid.x')
+ >>> headers, output = SimpleHandler(dataset3).dds()
+ >>> print ''.join(output)
+ Dataset {
+ Structure {
+ Int32 x[x = 2];
+ } grid;
+ } foo;
+ <BLANKLINE>
+
+ Short notation also works:
+
+ >>> dataset3 = constrain(dataset, 'x')
+ >>> headers, output = SimpleHandler(dataset3).dds()
+ >>> print ''.join(output)
+ Dataset {
+ Structure {
+ Int32 x[x = 2];
+ } grid;
+ } foo;
+ <BLANKLINE>
+
+ It also works with Sequences:
+
+ >>> dataset = DatasetType(name='foo')
+ >>> dataset['seq'] = SequenceType(name='seq')
+ >>> dataset['seq']['a'] = BaseType(name='a')
+ >>> dataset['seq']['b'] = BaseType(name='b')
+ >>> dataset['seq']['a'].data = range(5)
+ >>> dataset['seq']['b'].data = range(5,10)
+ >>> for i in dataset['seq'].data:
+ ... print i
+ (0, 5)
+ (1, 6)
+ (2, 7)
+ (3, 8)
+ (4, 9)
+ >>> dataset2 = constrain(dataset, 'seq.a')
+ >>> for i in dataset2['seq'].data:
+ ... print i
+ (0,)
+ (1,)
+ (2,)
+ (3,)
+ (4,)
+ >>> dataset2 = constrain(dataset, 'seq.b')
+ >>> for i in dataset2['seq'].data:
+ ... print i
+ (5,)
+ (6,)
+ (7,)
+ (8,)
+ (9,)
+ >>> dataset2 = constrain(dataset, 'seq.b,seq.a')
+ >>> for i in dataset2['seq'].data:
+ ... print i
+ (5, 0)
+ (6, 1)
+ (7, 2)
+ (8, 3)
+ (9, 4)
+
+ The function also parses selection expressions. Let's create a
+ dataset with sequential data:
+
+ >>> dataset = DatasetType(name='foo')
+ >>> dataset['seq'] = SequenceType(name='seq')
+ >>> dataset['seq']['index'] = BaseType(name='index', type='Int32')
+ >>> dataset['seq']['index'].data = [10, 11, 12, 13]
+ >>> dataset['seq']['temperature'] = BaseType(name='temperature', type='Float32')
+ >>> dataset['seq']['temperature'].data = [17.2, 15.1, 15.3, 15.1]
+ >>> dataset['seq']['site'] = BaseType(name='site', type='String')
+ >>> dataset['seq']['site'].data = ['Diamond_St', 'Blacktail_Loop', 'Platinum_St', 'Kodiak_Trail']
+
+ Here's the data:
+
+ >>> for i in dataset['seq'].data:
+ ... print i
+ (10, 17.199999999999999, 'Diamond_St')
+ (11, 15.1, 'Blacktail_Loop')
+ (12, 15.300000000000001, 'Platinum_St')
+ (13, 15.1, 'Kodiak_Trail')
+
+ Now suppose we only want data where ``index`` is greater than 11:
+
+ >>> dataset2 = constrain(dataset, 'seq&seq.index>11')
+ >>> for i in dataset2['seq'].data:
+ ... print i
+ (12, 15.300000000000001, 'Platinum_St')
+ (13, 15.1, 'Kodiak_Trail')
+
+ We can request only a few variables:
+
+ >>> dataset2 = constrain(dataset, 'seq.site&seq.index>11')
+ >>> for i in dataset2['seq'].data:
+ ... print i
+ ('Platinum_St',)
+ ('Kodiak_Trail',)
+
+ A few more tests:
+
+ >>> dataset = DatasetType(name='foo')
+ >>> dataset['a'] = StructureType(name='a')
+ >>> dataset['a']['shn'] = BaseType(name='shn')
+ >>> dataset['b'] = StructureType(name='b')
+ >>> dataset['b']['shn'] = BaseType(name='shn')
+ >>> dataset2 = constrain(dataset, 'a.shn')
+ >>> print dataset2 #doctest: +ELLIPSIS
+ {'a': {'shn': <dap.dtypes.BaseType object at ...>}}
+ >>> dataset3 = constrain(dataset, 'shn')
+ Traceback (most recent call last):
+ ...
+ ConstraintExpressionError: 'Ambiguous shorthand notation request: shn'
+ >>> dataset['shn'] = BaseType(name='shn')
+ >>> dataset3 = constrain(dataset, 'shn')
+ >>> print dataset3 #doctest: +ELLIPSIS
+ {'shn': <dap.dtypes.BaseType object at 0x1746290>}
+ """
+ # Parse constraints.
+ fields, queries = parse_querystring(constraints)
+
+ # Ids and names are used to check that requests made using the
+ # shorthand notation are not ambiguous. Used names are stored to
+ # make sure that at most only a single variables is returned from
+ # a given name.
+ ids = [var.id for var in walk(dataset)]
+ names = []
+
+ new = DatasetType(name=dataset.name, attributes=dataset.attributes.copy())
+ new = build(dataset, new, fields, queries, ids, names)
+
+ return new
+
+
+def build(dapvar, new, fields, queries, ids, names):
+ vars_ = fields.keys()
+ order = []
+ for var in dapvar.walk():
+ # Make a copy of the variable, so that later we can possibly add it
+ # to the dataset we're building (that's why it's a candidate).
+ candidate = copy.deepcopy(var)
+
+ # We first filter the data in sequences. This has to be done
+ # before variables are removed, since we can select values based
+ # on conditions on *other* variables. Eg: seq.a where seq.b > 1
+ if queries and isinstance(candidate, SequenceType):
+ # Filter candidate on the server-side, since the data may be
+ # proxied using ``dap.proxy.Proxy``.
+ candidate = candidate.filter(*queries)
+ # And then filter on the client side.
+ candidate = filter_(candidate, queries)
+
+ # If the variable was requested, either by id or name, or if no
+ # variables were requested, we simply add this candidate to the
+ # dataset we're building.
+ if not vars_ or candidate.id in vars_ or (candidate.name in vars_ and candidate.name not in ids):
+ new[candidate.name] = candidate
+
+ # Check if requests done using shn are not ambiguous.
+ if vars_ and candidate.id not in vars_: # request by shn
+ if candidate.name in names:
+ raise ConstraintExpressionError("Ambiguous shorthand notation request: %s" % candidate.name)
+ names.append(candidate.name)
+
+ # We also need to store the order in which the variables were
+ # requested. Later, we'll rearrange the variables in our built
+ # dataset in the correct order.
+ if vars_:
+ if candidate.id in vars_: index = vars_.index(candidate.id)
+ else: index = vars_.index(candidate.name)
+ order.append((index, candidate.name))
+
+ # If the variable was not requested, but it's a constructor, it's
+ # possible that one of its children has been requested. We apply
+ # the algorithm recursively on the variable.
+ elif not isinstance(var, BaseType):
+ # We clear the candidate after storing a copy with the filtered
+ # data and children. We will then append the requested children
+ # to the cleared candidate.
+ ccopy = copy.deepcopy(candidate)
+ if isinstance(candidate, StructureType):
+ candidate.clear()
+ else:
+ # If the variable is a grid we should return it as a
+ # structure with the requested fields.
+ parent = candidate._id[:-len(candidate.name)-1]
+ candidate = StructureType(name=candidate.name, attributes=candidate.attributes.copy())
+ candidate._set_id(parent)
+
+ # Check for requested children.
+ candidate = build(ccopy, candidate, fields, queries, ids, names)
+
+ # If the candidate has any keys, ie, stored variables, we add
+ # it to the dataset we are building.
+ if candidate.keys(): new[candidate.name] = candidate
+
+ # Check if we need to apply a slice in the variable.
+ slice_ = fields.get(candidate.id) or fields.get(candidate.name)
+ if slice_: candidate = slicevar(candidate, slice_)
+
+ # Sort variables according to order of requested variables.
+ if len(order) > 1:
+ order.sort()
+ new._keys = [item[1] for item in order]
+
+ return new
+
+
+def filter_(dapvar, queries):
+ # Get only the queries related to this variable.
+ queries_ = [q for q in queries if q.startswith(dapvar.id)]
+ if queries_:
+ # Build the filter and apply it to the data.
+ ids = [var.id for var in dapvar.values()]
+ f = buildfilter(queries_, ids)
+ data = itertools.ifilter(f, dapvar.data)
+
+ # Set the data in the stored variables.
+ data = list(data)
+ dapvar.data = data
+
+ return dapvar
+
+
+def slicevar(dapvar, slice_):
+ if slice_ != (slice(None),):
+ dapvar.data = dapvar.data[slice_]
+
+ try:
+ dapvar.shape = getattr(dapvar.data, 'shape', (len(dapvar.data),))
+ except TypeError:
+ pass
+
+ if isinstance(dapvar, GridType):
+ if not isiterable(slice_): slice_ = (slice_,)
+
+ # Slice the maps.
+ for map_,mapslice in zip(dapvar.maps.values(), slice_):
+ map_.data = map_.data[mapslice]
+ map_.shape = map_.data.shape
+
+ return dapvar
+
+
+def order(dataset, fields):
+ """
+ Order a given dataset according to the requested order.
+
+ >>> d = DatasetType(name='d')
+ >>> d['a'] = BaseType(name='a')
+ >>> d['b'] = BaseType(name='b')
+ >>> d['c'] = SequenceType(name='c')
+ >>> d['c']['d'] = BaseType(name='d')
+ >>> d['c']['e'] = BaseType(name='e')
+ >>> print order(d, 'b,c.e,c.d,a'.split(',')) #doctest: +ELLIPSIS
+ {'b': <dap.dtypes.BaseType object at ...>, 'c': {'e': <dap.dtypes.BaseType object at ...>, 'd': <dap.dtypes.BaseType object at ...>}, 'a': <dap.dtypes.BaseType object at ...>}
+ >>> print order(d, 'c.e,c.d,a'.split(',')) #doctest: +ELLIPSIS
+ {'c': {'e': <dap.dtypes.BaseType object at ...>, 'd': <dap.dtypes.BaseType object at ...>}, 'a': <dap.dtypes.BaseType object at ...>, 'b': <dap.dtypes.BaseType object at ...>}
+ >>> print order(d, 'b,c,a'.split(',')) #doctest: +ELLIPSIS
+ {'b': <dap.dtypes.BaseType object at ...>, 'c': {'d': <dap.dtypes.BaseType object at ...>, 'e': <dap.dtypes.BaseType object at ...>}, 'a': <dap.dtypes.BaseType object at ...>}
+ """
+ # Order the dataset.
+ dataset = copy.copy(dataset)
+ orders = []
+ n = len(dataset._keys)
+ for var in dataset.walk():
+ # Search for id first.
+ fields_ = [field[:len(var.id)] for field in fields]
+ if var.id in fields_: index = fields_.index(var.id)
+ # Else search by name.
+ elif var.name in fields: index = fields.index(var.name)
+ # Else preserve original order.
+ else: index = n + dataset._keys.index(var.name)
+ orders.append((index, var.name))
+
+ # Sort children.
+ if isinstance(var, StructureType):
+ dataset[var.name] = order(var, fields)
+
+ # Sort dataset.
+ if len(orders) > 1:
+ orders.sort()
+ dataset._keys = [item[1] for item in orders]
+
+ return dataset
+
+
+def walk(dapvar):
+ """
+ Iterate over all variables, including dapvar.
+ """
+ yield dapvar
+ try:
+ for child in dapvar.walk():
+ for var in walk(child): yield var
+ except:
+ pass
+
+
+def getslice(hyperslab, shape=None):
+ """Parse a hyperslab.
+
+ Parse a hyperslab to a slice according to variable shape. The hyperslab
+ follows the DAP specification, and ommited dimensions are returned in
+ their entirety.
+
+ >>> getslice('[0:1:2][0:1:2]')
+ (slice(0, 3, 1), slice(0, 3, 1))
+ >>> getslice('[0:2][0:2]')
+ (slice(0, 3, 1), slice(0, 3, 1))
+ >>> getslice('[0][2]')
+ (slice(0, 1, 1), slice(2, 3, 1))
+ >>> getslice('[0:1:1]')
+ (slice(0, 2, 1),)
+ >>> getslice('[0:2:1]')
+ (slice(0, 2, 2),)
+ >>> getslice('')
+ (slice(None, None, None),)
+ """
+ # Backwards compatibility. In pydap <= 2.2.3 the ``fields`` dict from
+ # helper.parse_querystring returned the slices as strings (instead of
+ # Python slices). These strings had to be passed to getslice to get a
+ # Python slice. Old plugins still do this, but with pydap >= 2.2.4
+ # they are already passing the slices, so we simply return them.
+ if not isinstance(hyperslab, basestring): return hyperslab or slice(None)
+
+ if hyperslab:
+ output = []
+ dimslices = hyperslab[1:-1].split('][')
+ for dimslice in dimslices:
+ start, size, step = _getsize(dimslice)
+ output.append(slice(start, start+size, step))
+ output = tuple(output)
+ else:
+ output = (slice(None),)
+
+ return output
+
+
+def _getsize(dimslice):
+ """Parse a dimension from a hyperslab.
+
+ Calculates the start, size and step from a DAP formatted hyperslab.
+
+ >>> _getsize('0:1:9')
+ (0, 10, 1)
+ >>> _getsize('0:2:9')
+ (0, 10, 2)
+ >>> _getsize('0')
+ (0, 1, 1)
+ >>> _getsize('0:9')
+ (0, 10, 1)
+ """
+ size = dimslice.split(':')
+
+ start = int(size[0])
+ if len(size) == 1:
+ stop = start
+ step = 1
+ elif len(size) == 2:
+ stop = int(size[1])
+ step = 1
+ elif len(size) == 3:
+ step = int(size[1])
+ stop = int(size[2])
+ else:
+ raise ConstraintExpressionError('Invalid hyperslab: %s.' % dimslice)
+ size = (stop-start) + 1
+
+ return start, size, step
+
+
+def buildfilter(queries, vars_):
+ """This function is a filter builder.
+
+ Given a list of DAP formatted queries and a list of variable names,
+ this function returns a dynamic filter function to filter rows.
+
+ From the example in the DAP specification:
+
+ >>> vars_ = ['index', 'temperature', 'site']
+ >>> data = []
+ >>> data.append([10, 17.2, 'Diamond_St'])
+ >>> data.append([11, 15.1, 'Blacktail_Loop'])
+ >>> data.append([12, 15.3, 'Platinum_St'])
+ >>> data.append([13, 15.1, 'Kodiak_Trail'])
+
+ Rows where index is greater-than-or-equal 11:
+
+ >>> f = buildfilter(['index>=11'], vars_)
+ >>> for line in itertools.ifilter(f, data):
+ ... print line
+ [11, 15.1, 'Blacktail_Loop']
+ [12, 15.300000000000001, 'Platinum_St']
+ [13, 15.1, 'Kodiak_Trail']
+
+ Rows where site ends with '_St':
+
+ >>> f = buildfilter(['site=~".*_St"'], vars_)
+ >>> for line in itertools.ifilter(f, data):
+ ... print line
+ [10, 17.199999999999999, 'Diamond_St']
+ [12, 15.300000000000001, 'Platinum_St']
+
+ Index greater-or-equal-than 11 AND site ends with '_St':
+
+ >>> f = buildfilter(['site=~".*_St"', 'index>=11'], vars_)
+ >>> for line in itertools.ifilter(f, data):
+ ... print line
+ [12, 15.300000000000001, 'Platinum_St']
+
+ Site is either 'Diamond_St' OR 'Blacktail_Loop':
+
+ >>> f = buildfilter(['site={"Diamond_St", "Blacktail_Loop"}'], vars_)
+ >>> for line in itertools.ifilter(f, data):
+ ... print line
+ [10, 17.199999999999999, 'Diamond_St']
+ [11, 15.1, 'Blacktail_Loop']
+
+ Index is either 10 OR 12:
+
+ >>> f = buildfilter(['index={10, 12}'], vars_)
+ >>> for line in itertools.ifilter(f, data):
+ ... print line
+ [10, 17.199999999999999, 'Diamond_St']
+ [12, 15.300000000000001, 'Platinum_St']
+
+ Python is great, isn't it? :)
+ """
+ filters = []
+ p = re.compile(r'''^ # Start of selection
+ {? # Optional { for multi-valued constants
+ (?P<var1>.*?) # Anything
+ }? # Closing }
+ (?P<op><=|>=|!=|=~|>|<|=) # Operators
+ {? # {
+ (?P<var2>.*?) # Anything
+ }? # }
+ $ # EOL
+ ''', re.VERBOSE)
+ for query in queries:
+ m = p.match(query)
+ if not m: raise ConstraintExpressionError('Invalid constraint expression: %s.' % query)
+
+ # Functions associated with each operator.
+ op = {'<' : operator.lt,
+ '>' : operator.gt,
+ '!=': operator.ne,
+ '=' : operator.eq,
+ '>=': operator.ge,
+ '<=': operator.le,
+ '=~': lambda a,b: re.match(b,a),
+ }[m.group('op')]
+ # Allow multiple comparisons in one line. Python rulez!
+ op = multicomp(op)
+
+ # Build the filter for the first variable.
+ if m.group('var1') in vars_:
+ i = vars_.index(m.group('var1'))
+ var1 = lambda L, i=i: operator.getitem(L, i)
+
+ # Build the filter for the second variable. It could be either
+ # a name or a constant.
+ if m.group('var2') in vars_:
+ i = vars_.index(m.group('var2'))
+ var2 = lambda L, i=i: operator.getitem(L, i)
+ else:
+ var2 = lambda x, m=m: expr_eval(m.group('var2'))
+
+ # This is the filter. We apply the function (op) to the variable
+ # filters (var1 and var2).
+ filter0 = lambda x, op=op, var1=var1, var2=var2: op(var1(x), var2(x))
+ filters.append(filter0)
+
+ if filters:
+ # We have to join all the filters that were built, using the AND
+ # operator. Believe me, this line does exactly that.
+ #
+ # You are not expected to understand this.
+ filter0 = lambda i: reduce(lambda x,y: x and y, [f(i) for f in filters])
+ else:
+ filter0 = bool
+
+ return filter0
+
+
+def multicomp(function):
+ """Multiple OR comparisons.
+
+ Given f(a,b), this function returns a new function g(a,b) which
+ performs multiple OR comparisons if b is a tuple.
+
+ >>> a = 1
+ >>> b = (0, 1, 2)
+ >>> operator.lt = multicomp(operator.lt)
+ >>> operator.lt(a, b)
+ True
+ """
+ def f(a, b):
+ if isinstance(b, tuple):
+ for i in b:
+ # Return True if any comparison is True.
+ if function(a, i): return True
+ return False
+ else:
+ return function(a, b)
+
+ return f
+
+
+def fix_slice(dims, index):
+ """Fix incomplete slices or slices with ellipsis.
+
+ The behaviour of this function was reversed-engineered from numpy.
+
+ >>> fix_slice(3, (0, Ellipsis, 0))
+ (0, slice(None, None, None), 0)
+ >>> fix_slice(4, (0, Ellipsis, 0))
+ (0, slice(None, None, None), slice(None, None, None), 0)
+ >>> fix_slice(4, (0, 0, Ellipsis, 0))
+ (0, 0, slice(None, None, None), 0)
+ >>> fix_slice(5, (0, Ellipsis, 0))
+ (0, slice(None, None, None), slice(None, None, None), slice(None, None, None), 0)
+ >>> fix_slice(5, (0, 0, Ellipsis, 0))
+ (0, 0, slice(None, None, None), slice(None, None, None), 0)
+ >>> fix_slice(5, (0, Ellipsis, 0, Ellipsis))
+ (0, slice(None, None, None), slice(None, None, None), 0, slice(None, None, None))
+ >>> fix_slice(4, slice(None, None, None))
+ (slice(None, None, None), slice(None, None, None), slice(None, None, None), slice(None, None, None))
+ >>> fix_slice(4, (slice(None, None, None), 0))
+ (slice(None, None, None), 0, slice(None, None, None), slice(None, None, None))
+ """
+ if not isinstance(index, tuple): index = (index,)
+
+ out = []
+ length = len(index)
+ for slice_ in index:
+ if slice_ is Ellipsis:
+ out.extend([slice(None)] * (dims - length + 1))
+ length += (dims - length)
+ else:
+ out.append(slice_)
+ index = tuple(out)
+
+ if len(index) < dims:
+ index += (slice(None),) * (dims - len(index))
+
+ return index
+
+
+def lenslice(slice_):
+ """
+ Return the number of values associated with a slice.
+
+ By Bob Drach.
+ """
+ step = slice_.step
+ if step is None: step = 1
+
+ if step > 0:
+ start = slice_.start
+ stop = slice_.stop
+ else:
+ start = slice_.stop
+ stop = slice_.start
+ step = -step
+ return ((stop-start-1)/step + 1)
+
+
+def parse_querystring(query):
+ """
+ Parse a query_string returning the requested variables, dimensions, and CEs.
+
+ >>> parse_querystring('a,b')
+ ({'a': (slice(None, None, None),), 'b': (slice(None, None, None),)}, [])
+ >>> parse_querystring('a[0],b[1]')
+ ({'a': (slice(0, 1, 1),), 'b': (slice(1, 2, 1),)}, [])
+ >>> parse_querystring('a[0],b[1]&foo.bar>1')
+ ({'a': (slice(0, 1, 1),), 'b': (slice(1, 2, 1),)}, ['foo.bar>1'])
+ >>> parse_querystring('a[0],b[1]&foo.bar>1&LAYERS=SST')
+ ({'a': (slice(0, 1, 1),), 'b': (slice(1, 2, 1),)}, ['foo.bar>1', 'LAYERS=SST'])
+ >>> parse_querystring('foo.bar>1&LAYERS=SST')
+ ({}, ['foo.bar>1', 'LAYERS=SST'])
+
+ """
+ if query is None: return {}, []
+
+ query = unquote(query)
+ constraints = query.split('&')
+
+ # Check if the first item is either a list of variables (projection)
+ # or a selection.
+ relops = ['<', '<=', '>', '>=', '=', '!=',' =~']
+ for relop in relops:
+ if relop in constraints[0]:
+ vars_ = []
+ queries = constraints[:]
+ break
+ else:
+ vars_ = constraints[0].split(',')
+ queries = constraints[1:]
+
+ fields = odict()
+ p = re.compile(r'(?P<name>[^[]+)(?P<shape>(\[[^\]]+\])*)')
+ for var in vars_:
+ if var:
+ # Check if the var has a slice.
+ c = p.match(var).groupdict()
+ id_ = quote(c['name'])
+ fields[id_] = getslice(c['shape'])
+
+ return fields, queries
+
+
+def escape_dods(dods, pad=''):
+ """
+ Escape a DODS response.
+
+ This is useful for debugging. You're probably spending too much time
+ with pydap if you need to use this.
+ """
+ if 'Data:\n' in dods:
+ index = dods.index('Data:\n') + len('Data:\n')
+ else:
+ index = 0
+
+ dds = dods[:index]
+ dods = dods[index:]
+
+ out = []
+ for i, char in enumerate(dods):
+ char = hex(ord(char))
+ char = char.replace('0x', '\\x')
+ if len(char) < 4: char = char.replace('\\x', '\\x0')
+ out.append(char)
+ if pad and (i%4 == 3): out.append(pad)
+ out = ''.join(out)
+ out = out.replace(r'\x5a\x00\x00\x00', '<start of sequence>')
+ out = out.replace(r'\xa5\x00\x00\x00', '<end of sequence>\n')
+ return dds + out
+
+
+def _test():
+ import doctest
+ doctest.testmod()
+
+if __name__ == "__main__":
+ _test()
Added: trunk/toolkits/basemap/lib/dap/lib.py
===================================================================
--- trunk/toolkits/basemap/lib/dap/lib.py (rev 0)
+++ trunk/toolkits/basemap/lib/dap/lib.py 2007-11-26 18:29:14 UTC (rev 4449)
@@ -0,0 +1,122 @@
+from __future__ import division
+
+"""Basic functions concerning the DAP.
+
+These functions are mostly related to encoding data according to the DAP.
+"""
+
+from urllib import quote as _quote
+
+__author__ = 'Roberto De Almeida <ro...@py...>'
+__version__ = (2,2,6,1) # module version
+__dap__ = (2,0) # protocol version
+
+# Constants that used to live in __init__.py but had to be moved
+# because we share the namespace with plugins and responses.
+USER_AGENT = 'pydap/%s' % '.'.join([str(_) for _ in __version__])
+INDENT = ' ' * 4
+VERBOSE = False
+CACHE = None
+TIMEOUT = None
+
+
+def isiterable(o):
+ """Tests if an object is iterable.
+
+ >>> print isiterable(range(10))
+ True
+ >>> print isiterable({})
+ True
+ >>> def a():
+ ... for i in range(10): yield i
+ >>> print isiterable(a())
+ True
+ >>> print isiterable('string')
+ False
+ >>> print isiterable(1)
+ False
+ """
+ # We DON'T want to iterate over strings.
+ if isinstance(o, basestring): return False
+
+ try:
+ iter(o)
+ return True
+ except TypeError:
+ return False
+
+
+def to_list(L):
+ if hasattr(L, 'tolist'): return L.tolist() # shortcut for numpy arrays
+ elif isiterable(L): return [to_list(item) for item in L]
+ else: return L
+
+
+def quote(name):
+ """Extended quote for the DAP spec.
+
+ The period MUST be escaped in names (DAP spec, item 5.1):
+
+ >>> quote("White space")
+ 'White%20space'
+ >>> _quote("Period.")
+ 'Period.'
+ >>> quote("Period.")
+ 'Period%2E'
+ """
+ return _quote(name).replace('.', '%2E')
+
+
+def encode_atom(atom):
+ r"""Atomic types encoding.
+
+ Encoding atomic types for the DAS. Integers should be printed using the
+ base 10 ASCII representation of its value:
+
...
[truncated message content] |
