Menu
▾
▴
[Netcdftools-commit] SF.net SVN: netcdftools:[67] trunk
|
From: <rob...@us...> - 2010-07-08 02:05:49
|
Revision: 67
http://netcdftools.svn.sourceforge.net/netcdftools/?rev=67&view=rev
Author: robertbridle
Date: 2010-07-08 02:05:42 +0000 (Thu, 08 Jul 2010)
Log Message:
-----------
ANDSWRON-703 - Created a program to convert netCDF files (split by latitude) into csv files.
Added Paths:
-----------
trunk/launch/NetCDF2CSVConverter.launch
trunk/src/main/java/au/csiro/netcdf/wron/MdbsyNetCDF2CSVConverter.java
trunk/src/test/java/au/csiro/netcdf/wron/TestMdbsyNetCDF2CSVConverter.java
Added: trunk/launch/NetCDF2CSVConverter.launch
===================================================================
--- trunk/launch/NetCDF2CSVConverter.launch (rev 0)
+++ trunk/launch/NetCDF2CSVConverter.launch 2010-07-08 02:05:42 UTC (rev 67)
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.eclipse.jdt.launching.localJavaApplication">
+<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
+<listEntry value="/netcdf-tools/src/main/java/au/csiro/netcdf/wron/MdbsyNetCDF2CSVConverter.java"/>
+</listAttribute>
+<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
+<listEntry value="1"/>
+</listAttribute>
+<stringAttribute key="org.eclipse.jdt.launching.CLASSPATH_PROVIDER" value="org.maven.ide.eclipse.launchconfig.classpathProvider"/>
+<stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="au.csiro.netcdf.wron.MdbsyNetCDF2CSVConverter"/>
+<stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="--inputDir "C:\test\netcdf\\" -outputDir "C:\test\csv\\""/>
+<stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="netcdf-tools"/>
+<stringAttribute key="org.eclipse.jdt.launching.SOURCE_PATH_PROVIDER" value="org.maven.ide.eclipse.launchconfig.sourcepathProvider"/>
+</launchConfiguration>
Added: trunk/src/main/java/au/csiro/netcdf/wron/MdbsyNetCDF2CSVConverter.java
===================================================================
--- trunk/src/main/java/au/csiro/netcdf/wron/MdbsyNetCDF2CSVConverter.java (rev 0)
+++ trunk/src/main/java/au/csiro/netcdf/wron/MdbsyNetCDF2CSVConverter.java 2010-07-08 02:05:42 UTC (rev 67)
@@ -0,0 +1,693 @@
+package au.csiro.netcdf.wron;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.RandomAccessFile;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.MissingOptionException;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.log4j.Logger;
+
+import ucar.ma2.Array;
+import ucar.ma2.InvalidRangeException;
+import ucar.nc2.NetcdfFile;
+import ucar.nc2.Variable;
+import ucar.nc2.units.DateFormatter;
+import au.csiro.netcdf.cli.CommandLineOptionsComparator;
+import au.csiro.netcdf.util.CSVTokenizer;
+import au.csiro.netcdf.util.Util;
+
+/**
+ * The class is a converter control class to convert Murray Darling Basin Sustainable Yields netCDF files into CSV
+ * files.
+ *
+ * Copyright 2010, CSIRO Australia
+ *
+ * @author Robert Bridle on 07/07/2010
+ * @version $Revision: {revision} $ $Date: 2008-10-31 11:24:49 +1100 (Fri, 31 Oct 2008) $
+ */
+public class MdbsyNetCDF2CSVConverter
+{
+ private static final String EPOC = "1895-01-01";
+
+ /**
+ * Constant that defines the logger to be used.
+ */
+ private static final Logger LOG = Logger
+ .getLogger(MdbsyNetCDF2CSVConverter.class.getName());
+
+ /**
+ * The latitude variable name used in the netCDF files.
+ */
+ private static final String LATITUDE_VARIABLE_NAME = "lat";
+
+ /**
+ * The longitude variable name used in the netCDF files.
+ */
+ private static final String LONGITUDE_VARIABLE_NAME = "long";
+
+ /**
+ * The time variable name used in the netCDF files.
+ */
+ private static final String TIME_VARIABLE_NAME = "time";
+
+ /**
+ * The number of milliseconds in a 24 hours.
+ */
+ private static final Long MILLISECONDS_IN_A_DAY = new Long("86400000"); // milliseconds in a day.
+
+ /**
+ * The date column text that is found in all csv files.
+ */
+ // private static final String DATE_COLUMN_TEXT = writeOutDateColumn();
+
+ /**
+ * The name of the command line option used for specifying the input directory containing the netCDF files.
+ */
+ public static final String INPUT_DIR = "inputDir";
+
+ /**
+ * The name of the command line option used for specifying the directory to create the csv files in.
+ */
+ public static final String OUTPUT_DIR = "outputDir";
+
+ /**
+ * The pattern for accepted file names where multiple files are accepted.
+ */
+ public static final String FILENAME_PATTERN = "pattern";
+
+ /**
+ * The column that the variable name is stored in the csv file.
+ */
+ private static final int VARIABLE_COLUMN = 0;
+
+ /**
+ * The only variables that we are interested in writing out to csv files are those that are in three dimensions
+ * (i.e. rank=3)
+ */
+ private static final int RANK_THREE = 3;
+
+ /**
+ * NetCDf file extension.
+ */
+ private static final Object CSV_FILE_EXTENSION = ".csv";
+
+ /**
+ * CSV file extension.
+ */
+ private static final Object NETCDF_FILE_EXTENSION = ".nc";
+
+ /**
+ * A string containing the date column text to be written out as the first column in all the csv files.
+ */
+ private static String DATE_COLUMN_TEXT = null;
+
+ /**
+ * @param args
+ */
+ @SuppressWarnings("static-access")
+ public static void main(String[] args) throws ParseException, IOException,
+ InvalidRangeException, java.text.ParseException
+ {
+ Options options = new Options();
+ try
+ {
+ Option inputDirectoryName = OptionBuilder
+ .withArgName("dir")
+ .hasArg()
+ .withDescription(
+ "1. the directory path containing the netCDF files to be converted.")
+ .isRequired(true).withLongOpt(INPUT_DIR).create("i");
+
+ Option outputDirectoryName = OptionBuilder
+ .withArgName("dir")
+ .hasArg()
+ .withDescription(
+ "2: the directory path to place the new csv files.")
+ .isRequired(true).withLongOpt(OUTPUT_DIR).create("o");
+
+ Option pattern = OptionBuilder
+ .withArgName("filename pattern")
+ .hasArg()
+ .withDescription(
+ "3: OPTIONAL, a pattern to match multiple existing files, where multiple files should be "
+ + "processed. If this option is present the outputFilename is expected to be a directory "
+ + "containing the files to be processed. The wildcard characters * and ? are supported but "
+ + "may need to be escaped by a \\ to avoid processing by the shell. e.g. \\*.nc ")
+ .isRequired(false).withLongOpt(FILENAME_PATTERN)
+ .create("p");
+
+ options.addOption(inputDirectoryName);
+ options.addOption(outputDirectoryName);
+ options.addOption(pattern);
+
+ // parse the command line arguments
+ CommandLine parsedCommandLine = new GnuParser()
+ .parse(options, args);
+
+ String inputDir = parsedCommandLine.getOptionValue(INPUT_DIR);
+ String outputDir = parsedCommandLine.getOptionValue(OUTPUT_DIR);
+
+ MdbsyNetCDF2CSVConverter converter = new MdbsyNetCDF2CSVConverter();
+
+ long start = System.currentTimeMillis();
+ converter.execute(inputDir, outputDir);
+ long end = System.currentTimeMillis() - start;
+ LOG.warn("Successfully converted all netcdf files to csv files in: " + end + " ms.");
+ }
+ catch (MissingOptionException moe)
+ {
+ LOG.error(moe.getMessage());
+
+ // generate the help/usage statement
+ String header = "Recreate ";
+ String footer = "\nExample: netcdf2csv -inputDirPath \"C:\\input\" -outputDirPath \"C:\\output\" \n"
+ + "Will convert the netCDF files in the input directory to csv file in the output directory.";
+ StringWriter sw = new StringWriter();
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.setOptionComparator(new CommandLineOptionsComparator());
+ formatter.printHelp(new PrintWriter(sw), 80, "-", header, options,
+ 0, 1, footer);
+ System.out.println(sw.toString());
+ }
+ }
+
+ /**
+ * Performs the conversion.
+ *
+ * @param inputDir
+ * the directory containing the Scenario A or C netCDF files.
+ * @param outputDir
+ * the directory in which the csv file will be placed.
+ * @throws InvalidRangeException
+ * thrown if an invalid range is attempted to be read from a netCDf variable.
+ * @throws IOException
+ */
+ public void execute(String inputDir, String outputDir) throws IOException,
+ InvalidRangeException
+ {
+ // perform an initial conversion from netCDF to csv, the csv file will look like this:
+ // Date, 1985-01-01, 1895-01-02, 1985-01-03, ...
+ // APET, 0.1, 0.3, 0.0, ....
+ // rainfall,0.0, 0.4, 0.1, ...
+ writeCSVFiles(inputDir, outputDir);
+
+ // transpose the csv files so the will look like this:
+ // Date, APET, rainfall
+ // 1895-01-01, 0.1, 0.0
+ // 1895-01-02, 0.3, 0.4
+ // 1895-01-03, 0.0, 0.1
+ transposeCSVFiles(outputDir);
+ }
+
+ /**
+ * For each netCDF file in the input directory:
+ * <p>
+ * Create a csv file for every latitude and longitude position.
+ * <p>
+ * Write out the value of a variable at every time interval, at a specific latitude and longitude position.
+ *
+ * @param inputDir
+ * the directory containing the Scenario A or C netCDF files.
+ * @param outputDir
+ * the directory in which the csv file will be placed.
+ * @throws InvalidRangeException
+ * thrown if an invalid range is attempted to be read from a netCDf variable.
+ * @throws IOException
+ */
+ private void writeCSVFiles(String inputDir, String outputDir)
+ throws IOException, InvalidRangeException
+ {
+ File dir = new File(inputDir);
+ File[] files = dir.listFiles();
+ if(files == null)
+ {
+ System.out.println("The input directory does not exist: " + inputDir);
+ LOG.error("The input directory does not exist: " + inputDir);
+ return;
+ }
+
+ for (int fileIndex = 0; fileIndex < files.length; fileIndex++)
+ {
+ if (files[fileIndex].isFile() && NETCDF_FILE_EXTENSION.equals(Util.getFileExtension(files[fileIndex].getName())))
+ {
+ NetcdfFile nc = null;
+ try
+ {
+ nc = NetcdfFile.open(files[fileIndex].getAbsolutePath());
+
+ Array latitudes = getLatitudeValues(nc);
+ Array longitudes = getLongitudeValues(nc);
+
+ LOG.info("latitude coordinate variable size: "
+ + latitudes.getSize());
+ LOG.info("longitude coordinate variable size: "
+ + longitudes.getSize());
+
+ for (int latIndex = 0; latIndex < latitudes.getSize(); latIndex++)
+ {
+ for (int longIndex = 0; longIndex < longitudes
+ .getSize(); longIndex++)
+ {
+ String fileName = createFileNameBasedOnLatLong(nc,
+ latIndex, longIndex);
+ String filePath = outputDir
+ + System.getProperty("file.separator")
+ + fileName + ".csv";
+
+ // if the csv file already exists, then append the next variable.
+ if (Util.fileExists(filePath))
+ {
+ File file = new File(filePath);
+ PrintWriter out = new PrintWriter(
+ new BufferedWriter(new FileWriter(file,
+ true /* append mode */)));
+
+ List<String> variablesAlreadyWrittenOut = getVariablesAlreadyWrittenOut(file);
+ List<String> variablesThatCanBeWrittenOut = getVariablesThatCanBeWrittenOut(nc);
+
+ variablesThatCanBeWrittenOut
+ .removeAll(variablesAlreadyWrittenOut);
+
+ try
+ {
+ writeOutVariable(nc, out,
+ variablesThatCanBeWrittenOut,
+ latIndex, longIndex);
+ }
+ finally
+ {
+ if (out != null)
+ out.close();
+ }
+ }
+ else
+ // if the csv file does not exist, create a new csv file and write all available variables.
+ {
+ PrintWriter out = new PrintWriter(new BufferedWriter(
+ new FileWriter(new File(filePath),
+ false /* append mode */)));
+
+ List<String> variablesThatCanBeWrittenOut = getVariablesThatCanBeWrittenOut(nc);
+
+ try
+ {
+ out.println(writeOutDateColumn(nc));
+
+ writeOutVariable(nc, out,
+ variablesThatCanBeWrittenOut,
+ latIndex, longIndex);
+ }
+ finally
+ {
+ if (out != null)
+ out.close();
+ }
+ }
+ }
+ }
+ }
+ catch (java.io.FileNotFoundException fnfe)
+ {
+ System.out.println("file not found= " + files[fileIndex]);
+ LOG.error(fnfe);
+ }
+ finally
+ {
+ if (nc != null)
+ nc.close();
+ }
+ }
+ }
+ }
+
+ /**
+ * Writes out the value of a variable at every time interval, at a specific latitude and longitude position.
+ *
+ * @throws InvalidRangeException
+ * @throws IOException
+ *
+ */
+ private void writeOutVariable(NetcdfFile nc, PrintWriter out,
+ List<String> variablesThatCanBeWrittenOut, int latIndex,
+ int longIndex) throws IOException, InvalidRangeException
+ {
+ for (String variableName : variablesThatCanBeWrittenOut)
+ {
+ Array subsection = getVariableAcrossTime(nc, variableName,
+ latIndex, longIndex);
+ out.println(variableName + ","
+ + subsection.toString().replaceAll(" ", ","));
+ }
+ }
+
+ /**
+ * Transpose the csv files so that columns represent a variable's value over time.
+ *
+ * @param outputDir
+ * the directory in which the csv file will be placed.
+ * @throws IOException
+ */
+ private void transposeCSVFiles(String outputDir) throws IOException
+ {
+ File dir = new File(outputDir);
+ File[] files = dir.listFiles();
+ if(files == null)
+ {
+ System.out.println("The output directory does not exist: " + outputDir);
+ LOG.error("The output directory does not exist: " + outputDir);
+ return;
+ }
+
+ for (int fileIndex = 0; fileIndex < files.length; fileIndex++)
+ {
+ if (files[fileIndex].isFile() && CSV_FILE_EXTENSION.equals(Util.getFileExtension(files[fileIndex].getName())))
+ {
+ String[][] matrix = readLookupFile(files[fileIndex]);
+
+ // write over-the-top of the csv file.
+ PrintWriter out = new PrintWriter(
+ new BufferedWriter(new FileWriter(files[fileIndex],
+ false /* append mode */)));
+
+ // transpose String[][] matrix and write back out to csv
+ String[][] transposed = transpose(matrix);
+
+ try
+ {
+ for (int row = 0; row < transposed.length; row++)
+ {
+ for (int col = 0; col < transposed[0].length; col++)
+ {
+ out.print(transposed[row][col]);
+ if(col < (transposed[0].length-1))
+ {
+ out.print(", ");
+ }
+ }
+ out.println();
+ }
+ }
+ finally
+ {
+ if (out != null)
+ out.close();
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns the names of the variables in the netCDF file that we need to write out for a given latitude x longitude
+ * position.
+ *
+ * @param nc
+ * a netCDF file {@link NetcdfFile}
+ * @return the names of the variables in the netCDF file that we need to write out for a given latitude x longitude
+ * position.
+ */
+ private List<String> getVariablesThatCanBeWrittenOut(NetcdfFile nc)
+ {
+ List<String> variablesThatCanBeWrittenOut = new ArrayList<String>();
+
+ for (Variable variable : nc.getVariables())
+ {
+ if (variable.getRank() == RANK_THREE)
+ {
+ variablesThatCanBeWrittenOut.add(variable.getName().trim());
+ }
+ }
+
+ return variablesThatCanBeWrittenOut;
+ }
+
+ /**
+ * Returns the name of the variables that have already be written out to a csv file.
+ *
+ * @param file
+ * the csv file being written out too.
+ * @return the name of the variables that have already be written out to a csv file.
+ * @throws IOException
+ */
+ private List<String> getVariablesAlreadyWrittenOut(File file)
+ throws IOException
+ {
+ List<String> variablesAlreadyWrittenOut = new ArrayList<String>();
+
+ String[][] matrix = readLookupFile(file);
+ for (int rows = 0; rows < matrix.length; rows++)
+ {
+ variablesAlreadyWrittenOut.add(matrix[rows][VARIABLE_COLUMN].trim());
+ }
+
+ return variablesAlreadyWrittenOut;
+ }
+
+ /**
+ * Creates a file name based on a given latitude and longitude.
+ *
+ * @param nc
+ * a netCDF file {@link NetcdfFile}
+ * @param latIndex
+ * the latitude index.
+ * @param longIndex
+ * the longitude index.
+ * @return a file name based on latitude and longitude position.
+ * @throws InvalidRangeException
+ * @throws IOException
+ */
+ private String createFileNameBasedOnLatLong(NetcdfFile nc, int latIndex,
+ int longIndex) throws IOException, InvalidRangeException
+ {
+ return "region_"
+ + nc.readSection("lat(" + latIndex + ":" + latIndex + ")")
+ .toString().trim()
+ + "_"
+ + nc.readSection("long(" + longIndex + ":" + longIndex + ")")
+ .toString().trim();
+ }
+
+ /**
+ * Parses a csv file into an row-by-column string matrix.
+ *
+ * @param file
+ * a csv file.
+ * @return row-by-column string matrix of the csv file.
+ * @throws IOException
+ */
+ public static String[][] readLookupFile(File file) throws IOException
+ {
+ RandomAccessFile raf = new RandomAccessFile(file, "r");
+ List<String[]> lineList = new ArrayList<String[]>();
+
+ try
+ {
+ String line = raf.readLine();
+ while (line != null)
+ {
+ String[] cols = new CSVTokenizer(line).getAllColumns();
+ lineList.add(cols);
+ line = raf.readLine();
+ }
+ }
+ finally
+ {
+ raf.close();
+ }
+
+ String[][] lineCols = lineList.toArray(new String[][] {});
+ return lineCols;
+ }
+
+ /**
+ * Transpose a matrix, e.g. [a][b] -> [b][a]
+ *
+ * @param values a String[][] matrix.
+ */
+ private String[][] transpose(String[][] values)
+ {
+ if(values.length == 0)
+ {
+ return new String[0][0];
+ }
+
+ String[][] transposed = new String[values[0].length][values.length];
+ for (int row = 0; row < values.length; row++)
+ {
+ for (int col = 0; col < values[0].length; col++)
+ {
+ transposed[col][row] = values[row][col];
+ }
+ }
+ return transposed;
+ }
+
+ /**
+ * Returns the value of a variable at every time interval, at a specific latitude and longitude position.
+ *
+ * @param nc
+ * a netCDF file {@link NetcdfFile}
+ * @param variableName
+ * the variable whose values are to be read.
+ * @param latIndex
+ * the latitude index.
+ * @param longIndex
+ * the longitude index.
+ * @return the value of a variable at every time interval, at a specific latitude and longitude position.
+ * @throws IOException
+ * @throws InvalidRangeException
+ */
+ private Array getVariableAcrossTime(NetcdfFile nc, String variableName,
+ int latIndex, int longIndex) throws IOException,
+ InvalidRangeException
+ {
+ try
+ {
+ return nc.readSection(variableName + "(" + latIndex + ":"
+ + latIndex + "," + longIndex + ":" + longIndex + ",:)");
+ }
+ catch (InvalidRangeException ire)
+ {
+ ire.printStackTrace();
+ LOG.error("Could not read section: " + variableName + "("
+ + latIndex + ":" + latIndex + "," + longIndex + ":"
+ + longIndex + ",:)", ire);
+ }
+ return null;
+ }
+
+ /**
+ * Returns the longitude coordinate variable values.
+ *
+ * @param nc
+ * a netCDF file {@link NetcdfFile}
+ * @return the longitude coordinate variable values.
+ * @throws IOException
+ * @throws IllegalArgumentException
+ */
+ private Array getLongitudeValues(NetcdfFile nc) throws IOException,
+ IllegalArgumentException
+ {
+ for (Variable v : nc.getVariables())
+ {
+ if (v.isCoordinateVariable()
+ && LONGITUDE_VARIABLE_NAME.equals(v.getName()))
+ {
+ return v.read();
+ }
+ }
+ LOG.error("Could not find coordinate variable: "
+ + LONGITUDE_VARIABLE_NAME);
+ throw new IllegalStateException("Could not find coordinate variable: "
+ + LONGITUDE_VARIABLE_NAME);
+ }
+
+ /**
+ * Returns the latitude coordinate variable values.
+ *
+ * @param nc
+ * a netCDF file {@link NetcdfFile}
+ * @return the latitude coordinate variable values.
+ * @throws IOException
+ * @throws IllegalArgumentException
+ */
+ private Array getLatitudeValues(NetcdfFile nc) throws IOException,
+ IllegalArgumentException
+ {
+ for (Variable v : nc.getVariables())
+ {
+ if (v.isCoordinateVariable()
+ && LATITUDE_VARIABLE_NAME.equals(v.getName()))
+ {
+ return v.read();
+ }
+ }
+ LOG.error("Could not find coordinate variable: "
+ + LATITUDE_VARIABLE_NAME);
+ throw new IllegalStateException("Could not find coordinate variable: "
+ + LATITUDE_VARIABLE_NAME);
+ }
+
+ /**
+ * Returns the time variable.
+ *
+ * @param nc
+ * a netCDF file {@link NetcdfFile}
+ * @return the time variable.
+ * @throws IllegalArgumentException
+ */
+ private static Variable getTimeVariable(NetcdfFile nc)
+ throws IllegalArgumentException
+ {
+ for (Variable v : nc.getVariables())
+ {
+ if (v.isCoordinateVariable()
+ && TIME_VARIABLE_NAME.equals(v.getName()))
+ {
+ return v;
+ }
+ }
+ LOG.error("Could not find coordinate variable: " + TIME_VARIABLE_NAME);
+ throw new IllegalStateException("Could not find coordinate variable: "
+ + TIME_VARIABLE_NAME);
+ }
+
+ /**
+ * Converts a "days since" time coordinate variable into a list of formatted dates.
+ *
+ * @param nc
+ * a netCDF file {@link NetcdfFile}
+ * @return a list of formatted dates.
+ */
+ private static String writeOutDateColumn(NetcdfFile nc)
+ {
+ // we only need to create the date column text once, it is the same for all csv files.
+ if (DATE_COLUMN_TEXT == null)
+ {
+ StringBuffer strBuf = new StringBuffer();
+
+ DateFormatter dateFormatter = new DateFormatter();
+ Date epoc;
+ try
+ {
+ epoc = dateFormatter.dateOnlyFormat(EPOC);
+ strBuf.append("Date,");
+
+ long timeVariableSize = getTimeVariable(nc).getSize();
+ for (int i = 0; i < timeVariableSize; i++)
+ {
+ strBuf.append(dateFormatter.toDateOnlyString(new Date(epoc
+ .getTime()
+ + (i * MILLISECONDS_IN_A_DAY))).trim());
+
+ if(i < (timeVariableSize-1))
+ {
+ strBuf.append(",");
+ }
+ }
+
+ DATE_COLUMN_TEXT = strBuf.toString();
+ }
+ catch (java.text.ParseException pe)
+ {
+ pe.printStackTrace();
+ LOG.error("Error parsing date", pe);
+ }
+
+ }
+ return DATE_COLUMN_TEXT;
+ }
+}
Added: trunk/src/test/java/au/csiro/netcdf/wron/TestMdbsyNetCDF2CSVConverter.java
===================================================================
--- trunk/src/test/java/au/csiro/netcdf/wron/TestMdbsyNetCDF2CSVConverter.java (rev 0)
+++ trunk/src/test/java/au/csiro/netcdf/wron/TestMdbsyNetCDF2CSVConverter.java 2010-07-08 02:05:42 UTC (rev 67)
@@ -0,0 +1,227 @@
+package au.csiro.netcdf.wron;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.util.ArrayList;
+
+import au.csiro.netcdf.NcWriteVariable;
+import au.csiro.netcdf.util.Util;
+
+import ucar.ma2.DataType;
+import ucar.nc2.Dimension;
+import ucar.nc2.NCdumpW;
+import ucar.nc2.NetcdfFile;
+import ucar.nc2.NetcdfFileWriteable;
+import ucar.nc2.Variable;
+import junit.framework.TestCase;
+
+/**
+ * This class is a unit test suite to verify that the {@link MdbsyNetCDF2CSVConverter} operates correctly.
+ *
+ * Copyright 2010, CSIRO Australia
+ *
+ * @author Robert Bridle on 08/07/2010
+ * @version $Revision: {revision} $ $Date: 2008-10-31 11:24:49 +1100 (Fri, 31 Oct 2008) $
+ */
+public class TestMdbsyNetCDF2CSVConverter extends TestCase
+{
+ /**
+ * The testing value for the latitude coordinate dimension.
+ */
+ private static final String DIM_LATITUDE = "lat";
+ private static final String VAR_LATITUDE = DIM_LATITUDE;
+ private static final String VAR_LATITUDE_VALUE = "-24.25";
+
+ /**
+ * The testing value for the longitude coordinate dimension.
+ */
+ private static final String DIM_LONGITUDE = "long";
+ private static final String VAR_LONGITUDE = DIM_LONGITUDE;
+ private static final String VAR_LONGITUDE_VALUE = "138.10";
+
+ /**
+ * The expected name of the converted csv file.
+ */
+ private static final String CONVERTED_CSV_FILE_NAME = System.getProperty("user.dir") + "\\region_-24.25_138.1.csv";
+
+ /**
+ * The testing value for the time coordinate dimension.
+ */
+ private static final String DIM_TIME = "time";
+ private static final String VAR_TIME = DIM_TIME;
+ private static final String VAR_TIME_VALUES = "0" + System.getProperty("line.separator") + "1"
+ + System.getProperty("line.separator") + "2" + System.getProperty("line.separator") + "3"
+ + System.getProperty("line.separator") + "4" + System.getProperty("line.separator") + "5"
+ + System.getProperty("line.separator") + "6" + System.getProperty("line.separator") + "7"
+ + System.getProperty("line.separator") + "8" + System.getProperty("line.separator") + "9";
+
+ /**
+ * The testing value for a variable defined over three coordinate dimensions (lat, long, time).
+ */
+ private static final String VAR_APET = "APET";
+ private static final String VAR_APET_VALUES = "0.000" + System.getProperty("line.separator") + "0.111"
+ + System.getProperty("line.separator") + "0.222" + System.getProperty("line.separator") + "0.333"
+ + System.getProperty("line.separator") + "0.444" + System.getProperty("line.separator") + "0.555"
+ + System.getProperty("line.separator") + "0.666" + System.getProperty("line.separator") + "0.777"
+ + System.getProperty("line.separator") + "0.888" + System.getProperty("line.separator") + "0.999";
+
+
+ /**
+ * The expected output converted csv structure.
+ */
+ private static String[] EXPECTED_CSV_FILE = { "Date, APET",
+ "1895-01-01, 0.0",
+ "1895-01-02, 0.111",
+ "1895-01-03, 0.222",
+ "1895-01-04, 0.333",
+ "1895-01-05, 0.444",
+ "1895-01-06, 0.555",
+ "1895-01-07, 0.666",
+ "1895-01-08, 0.777",
+ "1895-01-09, 0.888",
+ "1895-01-10, 0.999" };
+
+ /**
+ * The testing value for the dimension sizes.
+ */
+ private static final int LAT_DIM_SIZE = 1;
+ private static final int LONG_DIM_SIZE = 1;
+ private static final int TIME_DIM_SIZE = 10;
+
+ /**
+ * The testing value for the netCDF file to write to.
+ */
+ private static final String NC_FILE_NAME = System.getProperty("user.dir") + "\\ABC.nc";
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see junit.framework.TestCase#setUp()
+ */
+ @Override
+ protected void setUp() throws Exception
+ {
+ super.setUp();
+
+ // create dummy netCDF file with content.
+ createDummyNCFile(NC_FILE_NAME);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see junit.framework.TestCase#tearDown()
+ */
+ @Override
+ protected void tearDown() throws Exception
+ {
+ super.tearDown();
+
+ System.setOut(null);
+
+ // delete dummy netCDF file
+ File ncFile = new File(NC_FILE_NAME);
+ if (ncFile.exists())
+ {
+ NetcdfFile netcdfFile = NetcdfFile.open(NC_FILE_NAME);
+ netcdfFile.close();
+ ncFile.delete();
+ }
+
+ // delete converted csv file
+ File csvFile = new File(CONVERTED_CSV_FILE_NAME);
+ if (csvFile.exists())
+ {
+ csvFile.delete();
+ }
+ }
+
+ /**
+ * Test execute a valid NcWriteVariable command using Double datatype
+ */
+ public final void testNetCDF2CSVConversion() throws Exception
+ {
+ MdbsyNetCDF2CSVConverter.main(new String[] { "-i", System.getProperty("user.dir"), "-o",
+ System.getProperty("user.dir") });
+
+ // check that the converted csv file was created.
+ assertTrue("The expected converted csv file does not exist: " + CONVERTED_CSV_FILE_NAME, Util
+ .fileExists(CONVERTED_CSV_FILE_NAME));
+
+ String[][] csvMatrix = MdbsyNetCDF2CSVConverter.readLookupFile(new File(CONVERTED_CSV_FILE_NAME));
+
+ for (int row = 0; row < csvMatrix.length; row++)
+ {
+ StringBuffer line = new StringBuffer();
+ for (int col = 0; col < csvMatrix[0].length; col++)
+ {
+ line.append(csvMatrix[row][col]);
+ if(col < (csvMatrix[0].length-1))
+ {
+ line.append(", ");
+ }
+ }
+
+ assertEquals("The expected csv output is not correct.", EXPECTED_CSV_FILE[row], line.toString());
+ }
+ }
+
+ /**
+ * Makes a dummy netCDF file with known contents.
+ */
+ private void createDummyNCFile(String outputFilename) throws IOException
+ {
+ NetcdfFileWriteable ncFile = null;
+
+ ncFile = NetcdfFileWriteable.createNew(outputFilename, true);
+ try
+ {
+ Dimension dimension = new Dimension(DIM_LATITUDE, LAT_DIM_SIZE, true /* isShared */,
+ false /* isUnlimited */, false /* isVariableLength */);
+ ncFile.addDimension(null, dimension);
+
+ dimension = new Dimension(DIM_LONGITUDE, LONG_DIM_SIZE, true /* isShared */, false /* isUnlimited */, false /* isVariableLength */);
+ ncFile.addDimension(null, dimension);
+
+ dimension = new Dimension(DIM_TIME, TIME_DIM_SIZE, true /* isShared */, false /* isUnlimited */, false /* isVariableLength */);
+ ncFile.addDimension(null, dimension);
+
+ // define coordinate variables.
+ ncFile.addVariable(VAR_LATITUDE, DataType.FLOAT, DIM_LATITUDE);
+ ncFile.addVariable(VAR_LONGITUDE, DataType.FLOAT, DIM_LONGITUDE);
+ ncFile.addVariable(VAR_TIME, DataType.INT, DIM_TIME);
+
+ // variable is defined over three coordinates dimensions.
+ ncFile.addVariable(VAR_APET, DataType.FLOAT, DIM_LATITUDE + " " + DIM_LONGITUDE + " " + DIM_TIME);
+ ncFile.create();
+ }
+ finally
+ {
+ ncFile.close();
+ }
+
+ // populate the coordinate variables with string of int values.
+ NcWriteVariable command = new NcWriteVariable();
+ command.execute(new File(outputFilename), VAR_LATITUDE, "0-" + String.valueOf(LAT_DIM_SIZE - 1),
+ new ByteArrayInputStream(VAR_LATITUDE_VALUE.getBytes("UTF-8")), false);
+ command.execute(new File(outputFilename), VAR_LONGITUDE, "0-" + String.valueOf(LONG_DIM_SIZE - 1),
+ new ByteArrayInputStream(VAR_LONGITUDE_VALUE.getBytes("UTF-8")), false);
+ command.execute(new File(outputFilename), VAR_TIME, "0-" + String.valueOf(TIME_DIM_SIZE - 1),
+ new ByteArrayInputStream(VAR_TIME_VALUES.getBytes("UTF-8")), false);
+
+ try
+ {
+ command.execute(new File(outputFilename), VAR_APET, "0-" + String.valueOf(LAT_DIM_SIZE - 1) + ", 0-"
+ + String.valueOf(LONG_DIM_SIZE - 1) + ", 0-" + String.valueOf(TIME_DIM_SIZE - 1),
+ new ByteArrayInputStream(VAR_APET_VALUES.getBytes("UTF-8")), false);
+ }
+ catch (IllegalArgumentException iae)
+ {
+ iae.printStackTrace();
+ fail("could not create file: " + outputFilename + " to being testing of netCDF 2 CSV converter");
+ }
+ }
+}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
