Menu
▾
▴
matplotlib-checkins
|
From: <js...@us...> - 2007-11-24 19:43:39
|
Revision: 4431
http://matplotlib.svn.sourceforge.net/matplotlib/?rev=4431&view=rev
Author: jswhit
Date: 2007-11-24 11:43:37 -0800 (Sat, 24 Nov 2007)
Log Message:
-----------
modify pupynere to create masked array when missing_value
and/or _FillValue attribute is defined, and apply scale_factor
and add_offset.
Modified Paths:
--------------
trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py
Modified: trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py
===================================================================
--- trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py 2007-11-24 13:25:06 UTC (rev 4430)
+++ trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py 2007-11-24 19:43:37 UTC (rev 4431)
@@ -31,7 +31,7 @@
import itertools
import mmap
-from numpy import ndarray, zeros, array
+from numpy import ndarray, zeros, array, ma, squeeze
ABSENT = '\x00' * 8
@@ -257,7 +257,23 @@
}
def __getitem__(self, index):
- return self.__array_data__.__getitem__(index)
+ # modified by jsw to automatically
+ # - remove singleton dimensions
+ # - create a masked array using missing_value or _FillValue attribute
+ # - apply scale_factor and add_offset to packed integer data
+ datout = squeeze(self.__array_data__.__getitem__(index))
+ try:
+ datout = ma.masked_values(datout, self.missing_value)
+ except:
+ try:
+ datout = ma.masked_values(datout, self._FillValue)
+ except:
+ pass
+ try:
+ datout = self.scale_factor*datout + self.add_offset
+ except:
+ pass
+ return datout
def getValue(self):
"""For scalars."""
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <js...@us...> - 2007-11-26 19:21:27
|
Revision: 4456
http://matplotlib.svn.sourceforge.net/matplotlib/?rev=4456&view=rev
Author: jswhit
Date: 2007-11-26 11:21:23 -0800 (Mon, 26 Nov 2007)
Log Message:
-----------
add docstring.
Modified Paths:
--------------
trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py
Modified: trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py
===================================================================
--- trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py 2007-11-26 19:14:29 UTC (rev 4455)
+++ trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py 2007-11-26 19:21:23 UTC (rev 4456)
@@ -52,10 +52,13 @@
_typecodes = dict([[_v,_k] for _k,_v in typemap.items()])
def NetCDFFile(file):
+ """NetCDF File reader. API is the same as Scientific.IO.NetCDF.
+ if 'file' is a URL that starts with 'http', the pydap client is
+ is used to read the data over http."""
if file.startswith('http'):
- return RemoteFile(file)
+ return _RemoteFile(file)
else:
- return LocalFile(file)
+ return _LocalFile(file)
def _maskandscale(var,datout):
if hasattr(var, 'missing_value') and (datout == var.missing_value).any():
@@ -68,7 +71,7 @@
pass
return datout
-class RemoteFile(object):
+class _RemoteFile(object):
"""A NetCDF file reader. API is the same as Scientific.IO.NetCDF."""
def __init__(self, file):
@@ -108,13 +111,13 @@
for k,d in self._buffer.iteritems():
if isinstance(d, GridType) or isinstance(d, ArrayType):
name = k
- self.variables[name] = RemoteVariable(d)
+ self.variables[name] = _RemoveVariable(d)
def close(self):
self._buffer.close()
-class RemoteVariable(object):
+class _RemoveVariable(object):
def __init__(self, var):
self._var = var
self.dtype = var.type
@@ -134,7 +137,7 @@
return _typecodes[self.dtype]
-class LocalFile(object):
+class _LocalFile(object):
"""A NetCDF file reader. API is the same as Scientific.IO.NetCDF."""
def __init__(self, file):
@@ -266,7 +269,7 @@
# Read offset.
begin = [self._unpack_int, self._unpack_int64][self.version_byte-1]()
- return LocalVariable(self._buffer.fileno(), nc_type, vsize, begin, shape, dimensions, attributes, isrec, self._recsize)
+ return _LocalVariable(self._buffer.fileno(), nc_type, vsize, begin, shape, dimensions, attributes, isrec, self._recsize)
def _read_values(self, n, nc_type):
bytes = [1, 1, 2, 4, 4, 8]
@@ -305,7 +308,7 @@
self._buffer.close()
-class LocalVariable(object):
+class _LocalVariable(object):
def __init__(self, fileno, nc_type, vsize, begin, shape, dimensions, attributes, isrec=False, recsize=0):
self._nc_type = nc_type
self._vsize = vsize
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <js...@us...> - 2007-11-26 22:53:39
|
Revision: 4458
http://matplotlib.svn.sourceforge.net/matplotlib/?rev=4458&view=rev
Author: jswhit
Date: 2007-11-26 14:53:37 -0800 (Mon, 26 Nov 2007)
Log Message:
-----------
make close method a no-op for remote datasets.
Modified Paths:
--------------
trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py
Modified: trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py
===================================================================
--- trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py 2007-11-26 21:53:01 UTC (rev 4457)
+++ trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py 2007-11-26 22:53:37 UTC (rev 4458)
@@ -114,7 +114,8 @@
self.variables[name] = _RemoveVariable(d)
def close(self):
- self._buffer.close()
+ # this is a no-op provided for compatibility
+ pass
class _RemoveVariable(object):
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <js...@us...> - 2007-11-26 23:06:29
|
Revision: 4459
http://matplotlib.svn.sourceforge.net/matplotlib/?rev=4459&view=rev
Author: jswhit
Date: 2007-11-26 15:06:15 -0800 (Mon, 26 Nov 2007)
Log Message:
-----------
docstring modifications.
Modified Paths:
--------------
trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py
Modified: trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py
===================================================================
--- trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py 2007-11-26 22:53:37 UTC (rev 4458)
+++ trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py 2007-11-26 23:06:15 UTC (rev 4459)
@@ -53,8 +53,11 @@
def NetCDFFile(file):
"""NetCDF File reader. API is the same as Scientific.IO.NetCDF.
- if 'file' is a URL that starts with 'http', the pydap client is
- is used to read the data over http."""
+ If 'file' is a URL that starts with 'http', it is assumed
+ to be a remote OPenDAP dataest, and the python dap client is used
+ to retrieve the data. Only the OPenDAP Array and Grid data
+ types are recognized. If file does not start with 'http', it
+ is assumed to be a local NetCDF file."""
if file.startswith('http'):
return _RemoteFile(file)
else:
@@ -361,8 +364,3 @@
def typecode(self):
return ['b', 'c', 'h', 'i', 'f', 'd'][self._nc_type-1]
-
-
-def _test():
- import doctest
- doctest.testmod()
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <js...@us...> - 2007-11-26 23:08:56
|
Revision: 4460
http://matplotlib.svn.sourceforge.net/matplotlib/?rev=4460&view=rev
Author: jswhit
Date: 2007-11-26 15:08:55 -0800 (Mon, 26 Nov 2007)
Log Message:
-----------
fix typo.
Modified Paths:
--------------
trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py
Modified: trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py
===================================================================
--- trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py 2007-11-26 23:06:15 UTC (rev 4459)
+++ trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py 2007-11-26 23:08:55 UTC (rev 4460)
@@ -114,14 +114,14 @@
for k,d in self._buffer.iteritems():
if isinstance(d, GridType) or isinstance(d, ArrayType):
name = k
- self.variables[name] = _RemoveVariable(d)
+ self.variables[name] = _RemoteVariable(d)
def close(self):
# this is a no-op provided for compatibility
pass
-class _RemoveVariable(object):
+class _RemoteVariable(object):
def __init__(self, var):
self._var = var
self.dtype = var.type
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <js...@us...> - 2007-11-27 18:05:28
|
Revision: 4470
http://matplotlib.svn.sourceforge.net/matplotlib/?rev=4470&view=rev
Author: jswhit
Date: 2007-11-27 10:05:25 -0800 (Tue, 27 Nov 2007)
Log Message:
-----------
make automatic masking and rescaling of variable data optional
in NetCDFFile.
Modified Paths:
--------------
trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py
Modified: trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py
===================================================================
--- trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py 2007-11-27 17:40:45 UTC (rev 4469)
+++ trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py 2007-11-27 18:05:25 UTC (rev 4470)
@@ -56,7 +56,7 @@
_typecodes = dict([[_v,_k] for _k,_v in typemap.items()])
-def NetCDFFile(file):
+def NetCDFFile(file, maskandscale=True):
"""NetCDF File reader. API is the same as Scientific.IO.NetCDF.
If 'file' is a URL that starts with 'http', it is assumed
to be a remote OPenDAP dataset, and the python dap client is used
@@ -66,9 +66,18 @@
with a pure python NetCDF reader, otherwise PyNIO
(http://www.pyngl.ucar.edu/Nio.shtml) will be used (if it is installed).
PyNIO supports NetCDF version 4, GRIB1, GRIB2, HDF4 and HDFEOS2 files.
+ Data read from OPenDAP and NetCDF version 3 datasets will
+ automatically be converted to masked arrays if the variable has either
+ a 'missing_value' or '_FillValue' attribute, and some data points
+ are equal to the value specified by that attribute. In addition,
+ variables stored as integers that have the 'scale_factor' and
+ 'add_offset' attribute will automatically be rescaled to floats when
+ read. If PyNIO is used, neither of the automatic conversions will
+ be performed. To suppress these automatic conversions, set the
+ maskandscale keyword to False.
"""
if file.startswith('http'):
- return _RemoteFile(file)
+ return _RemoteFile(file,maskandscale)
else:
# use pynio if it is installed and the file cannot
# be read with the pure python netCDF reader. This allows
@@ -76,13 +85,13 @@
# to be read.
if has_pynio:
try:
- f = _LocalFile(file)
+ f = _LocalFile(file,maskandscale)
except:
f = nio.open_file(file)
# otherwise, use the pupynere netCDF 3 pure python reader.
# (will fail if file is not a netCDF version 3 file).
else:
- f = _LocalFile(file)
+ f = _LocalFile(file,maskandscale)
return f
def _maskandscale(var,datout):
@@ -99,8 +108,9 @@
class _RemoteFile(object):
"""A NetCDF file reader. API is the same as Scientific.IO.NetCDF."""
- def __init__(self, file):
+ def __init__(self, file, maskandscale):
self._buffer = open_remote(file)
+ self._maskandscale = maskandscale
self._parse()
def read(self, size=-1):
@@ -136,7 +146,7 @@
for k,d in self._buffer.iteritems():
if isinstance(d, GridType) or isinstance(d, ArrayType):
name = k
- self.variables[name] = _RemoteVariable(d)
+ self.variables[name] = _RemoteVariable(d,self._maskandscale)
def close(self):
# this is a no-op provided for compatibility
@@ -144,8 +154,9 @@
class _RemoteVariable(object):
- def __init__(self, var):
+ def __init__(self, var, maskandscale):
self._var = var
+ self._maskandscale = maskandscale
self.dtype = var.type
self.shape = var.shape
self.dimensions = var.dimensions
@@ -157,7 +168,10 @@
# - remove singleton dimensions
# - create a masked array using missing_value or _FillValue attribute
# - apply scale_factor and add_offset to packed integer data
- return _maskandscale(self,datout)
+ if self._maskandscale:
+ return _maskandscale(self,datout)
+ else:
+ return datout
def typecode(self):
return _typecodes[self.dtype]
@@ -166,8 +180,9 @@
class _LocalFile(object):
"""A NetCDF file reader. API is the same as Scientific.IO.NetCDF."""
- def __init__(self, file):
+ def __init__(self, file, maskandscale):
self._buffer = open(file, 'rb')
+ self._maskandscale = maskandscale
self._parse()
def read(self, size=-1):
@@ -295,7 +310,7 @@
# Read offset.
begin = [self._unpack_int, self._unpack_int64][self.version_byte-1]()
- return _LocalVariable(self._buffer.fileno(), nc_type, vsize, begin, shape, dimensions, attributes, isrec, self._recsize)
+ return _LocalVariable(self._buffer.fileno(), nc_type, vsize, begin, shape, dimensions, attributes, isrec, self._recsize, maskandscale=self._maskandscale)
def _read_values(self, n, nc_type):
bytes = [1, 1, 2, 4, 4, 8]
@@ -335,7 +350,7 @@
class _LocalVariable(object):
- def __init__(self, fileno, nc_type, vsize, begin, shape, dimensions, attributes, isrec=False, recsize=0):
+ def __init__(self, fileno, nc_type, vsize, begin, shape, dimensions, attributes, isrec=False, recsize=0, maskandscale=True):
self._nc_type = nc_type
self._vsize = vsize
self._begin = begin
@@ -344,6 +359,7 @@
self.attributes = attributes # for ``dap.plugins.netcdf``
self.__dict__.update(attributes)
self._is_record = isrec
+ self._maskandscale = maskandscale
# Number of bytes and type.
self._bytes = [1, 1, 2, 4, 4, 8][self._nc_type-1]
@@ -378,7 +394,10 @@
# - remove singleton dimensions
# - create a masked array using missing_value or _FillValue attribute
# - apply scale_factor and add_offset to packed integer data
- return _maskandscale(self,datout)
+ if self._maskandscale:
+ return _maskandscale(self,datout)
+ else:
+ return datout
def getValue(self):
"""For scalars."""
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <js...@us...> - 2007-12-12 14:00:22
|
Revision: 4704
http://matplotlib.svn.sourceforge.net/matplotlib/?rev=4704&view=rev
Author: jswhit
Date: 2007-12-12 06:00:04 -0800 (Wed, 12 Dec 2007)
Log Message:
-----------
make sure masked arrays have correct fill_value
Modified Paths:
--------------
trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py
Modified: trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py
===================================================================
--- trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py 2007-12-12 13:55:12 UTC (rev 4703)
+++ trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py 2007-12-12 14:00:04 UTC (rev 4704)
@@ -96,9 +96,9 @@
def _maskandscale(var,datout):
if hasattr(var, 'missing_value') and (datout == var.missing_value).any():
- datout = ma.masked_values(datout, var.missing_value)
+ datout = ma.masked_array(datout, mask = datout == var.missing_value, fill_value=var.missing_value)
elif hasattr(var, '_FillValue') and (datout == var._FillValue).any():
- datout = ma.masked_values(datout, var._FillValue)
+ datout = ma.masked_array(datout, mask = datout == var._FillValue, fill_value=var._FillValue)
try:
datout = var.scale_factor*datout + var.add_offset
except:
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <js...@us...> - 2007-12-12 16:47:54
|
Revision: 4709
http://matplotlib.svn.sourceforge.net/matplotlib/?rev=4709&view=rev
Author: jswhit
Date: 2007-12-12 08:25:57 -0800 (Wed, 12 Dec 2007)
Log Message:
-----------
break some long lines
Modified Paths:
--------------
trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py
Modified: trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py
===================================================================
--- trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py 2007-12-12 15:01:20 UTC (rev 4708)
+++ trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py 2007-12-12 16:25:57 UTC (rev 4709)
@@ -96,9 +96,11 @@
def _maskandscale(var,datout):
if hasattr(var, 'missing_value') and (datout == var.missing_value).any():
- datout = ma.masked_array(datout, mask = datout == var.missing_value, fill_value=var.missing_value)
+ datout = ma.masked_array(datout,mask=datout==var.missing_value,
+ fill_value=var.missing_value)
elif hasattr(var, '_FillValue') and (datout == var._FillValue).any():
- datout = ma.masked_array(datout, mask = datout == var._FillValue, fill_value=var._FillValue)
+ datout = ma.masked_array(datout,mask=datout==var._FillValue,
+ fill_value=var._FillValue)
try:
datout = var.scale_factor*datout + var.add_offset
except:
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <js...@us...> - 2007-12-21 18:53:30
|
Revision: 4786
http://matplotlib.svn.sourceforge.net/matplotlib/?rev=4786&view=rev
Author: jswhit
Date: 2007-12-21 10:53:29 -0800 (Fri, 21 Dec 2007)
Log Message:
-----------
return masked array if data equals default netcdf _FillValue for that
data type.
Modified Paths:
--------------
trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py
Modified: trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py
===================================================================
--- trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py 2007-12-21 16:22:42 UTC (rev 4785)
+++ trunk/toolkits/basemap/lib/matplotlib/toolkits/basemap/pupynere.py 2007-12-21 18:53:29 UTC (rev 4786)
@@ -31,7 +31,8 @@
import itertools
import mmap
-from numpy import ndarray, empty, array, ma, squeeze
+from numpy import ndarray, empty, array, ma, squeeze, zeros
+import numpy
from dap.client import open as open_remote
from dap.dtypes import ArrayType, GridType, typemap
@@ -55,6 +56,19 @@
NC_ATTRIBUTE = '\x00\x00\x00\x0c'
_typecodes = dict([[_v,_k] for _k,_v in typemap.items()])
+# default _FillValue for netcdf types (apply also to corresponding
+# DAP types).
+_default_fillvals = {'c':'\0',
+ 'S':"",
+ 'b':-127,
+ 'B':-127,
+ 'h':-32767,
+ 'H':65535,
+ 'i':-2147483647L,
+ 'L':4294967295L,
+ 'q':-2147483647L,
+ 'f':9.9692099683868690e+36,
+ 'd':9.9692099683868690e+36}
def NetCDFFile(file, maskandscale=True):
"""NetCDF File reader. API is the same as Scientific.IO.NetCDF.
@@ -95,12 +109,21 @@
return f
def _maskandscale(var,datout):
+ totalmask = zeros(datout.shape,numpy.bool)
+ fillval = None
if hasattr(var, 'missing_value') and (datout == var.missing_value).any():
- datout = ma.masked_array(datout,mask=datout==var.missing_value,
- fill_value=var.missing_value)
- elif hasattr(var, '_FillValue') and (datout == var._FillValue).any():
- datout = ma.masked_array(datout,mask=datout==var._FillValue,
- fill_value=var._FillValue)
+ fillval = var.missing_value
+ totalmask = totalmask + datout==fillval
+ if hasattr(var, '_FillValue') and (datout == var._FillValue).any():
+ if fillval is None:
+ fillval = var._FillValue
+ totalmask = totalmask + datout==var._FillValue
+ elif (datout == _default_fillvals[var.typecode()]).any():
+ if fillval is None:
+ fillval = _default_fillvals[var.typecode()]
+ totalmask = totalmask + datout==_default_fillvals[var.dtype]
+ if fillval is not None:
+ datout = ma.masked_array(datout,mask=totalmask,fill_value=fillval)
try:
datout = var.scale_factor*datout + var.add_offset
except:
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
×
Want the latest updates on software, tech news, and AI?
Get latest updates about software, tech news, and AI from SourceForge directly in your inbox once a month.
Thanks for helping keep SourceForge clean.
X