Menu
▾
▴
[flatpack-svn] SF.net SVN: flatpack:[402] trunk/flatpack/src
|
From: <zep...@us...> - 2008-09-01 13:20:57
|
Revision: 402
http://flatpack.svn.sourceforge.net/flatpack/?rev=402&view=rev
Author: zepernick
Date: 2008-09-01 13:21:06 +0000 (Mon, 01 Sep 2008)
Log Message:
-----------
added new option on the Parser to store the raw data of each line.
Modified Paths:
--------------
trunk/flatpack/src/main/java/net/sf/flatpack/AbstractDelimiterParser.java
trunk/flatpack/src/main/java/net/sf/flatpack/AbstractFixedLengthParser.java
trunk/flatpack/src/main/java/net/sf/flatpack/AbstractParser.java
trunk/flatpack/src/main/java/net/sf/flatpack/DataSet.java
trunk/flatpack/src/main/java/net/sf/flatpack/DefaultDataSet.java
trunk/flatpack/src/main/java/net/sf/flatpack/Parser.java
trunk/flatpack/src/main/java/net/sf/flatpack/brparse/BuffReaderDelimParser.java
trunk/flatpack/src/main/java/net/sf/flatpack/brparse/BuffReaderFixedParser.java
trunk/flatpack/src/main/java/net/sf/flatpack/structure/Row.java
trunk/flatpack/src/test/java/net/sf/flatpack/pzparser/PZParserOptsTest.java
Modified: trunk/flatpack/src/main/java/net/sf/flatpack/AbstractDelimiterParser.java
===================================================================
--- trunk/flatpack/src/main/java/net/sf/flatpack/AbstractDelimiterParser.java 2008-08-31 12:43:32 UTC (rev 401)
+++ trunk/flatpack/src/main/java/net/sf/flatpack/AbstractDelimiterParser.java 2008-09-01 13:21:06 UTC (rev 402)
@@ -190,15 +190,21 @@
row.setMdkey(mdkey.equals(FPConstants.DETAIL_ID) ? null : mdkey); // try
// to limit the memory use
row.setCols(columns);
- row.setRowNumber(lineCount);
- /** add the row to the array */
- ds.addRow(row);
-
+ row.setRowNumber(lineCount);
if (isFlagEmptyRows()) {
//user has elected to have the parser flag rows that are empty
row.setEmpty(ParserUtils.isListElementsEmpty(columns));
}
+ if (isStoreRawDataToDataSet()) {
+ //user told the parser to keep a copy of the raw data in the row
+ //WARNING potential for high memory usage here
+ row.setRawData(line);
+ }
+ //add the row to the array
+ ds.addRow(row);
+
+
}
} finally {
if (br != null) {
Modified: trunk/flatpack/src/main/java/net/sf/flatpack/AbstractFixedLengthParser.java
===================================================================
--- trunk/flatpack/src/main/java/net/sf/flatpack/AbstractFixedLengthParser.java 2008-08-31 12:43:32 UTC (rev 401)
+++ trunk/flatpack/src/main/java/net/sf/flatpack/AbstractFixedLengthParser.java 2008-09-01 13:21:06 UTC (rev 402)
@@ -146,6 +146,11 @@
//user has elected to have the parser flag rows that are empty
row.setEmpty(ParserUtils.isListElementsEmpty(row.getCols()));
}
+ if (isStoreRawDataToDataSet()) {
+ //user told the parser to keep a copy of the raw data in the row
+ //WARNING potential for high memory usage here
+ row.setRawData(line);
+ }
// add the row to the array
ds.addRow(row);
Modified: trunk/flatpack/src/main/java/net/sf/flatpack/AbstractParser.java
===================================================================
--- trunk/flatpack/src/main/java/net/sf/flatpack/AbstractParser.java 2008-08-31 12:43:32 UTC (rev 401)
+++ trunk/flatpack/src/main/java/net/sf/flatpack/AbstractParser.java 2008-09-01 13:21:06 UTC (rev 402)
@@ -73,6 +73,8 @@
private boolean flagEmptyRows;
private boolean storeRawDataToDataError;
+
+ private boolean storeRawDataToDataSet;
protected AbstractParser(final Reader dataSourceReader) {
this.dataSourceReader = dataSourceReader;
@@ -293,4 +295,18 @@
public void setStoreRawDataToDataError(boolean storeRawDataToDataError) {
this.storeRawDataToDataError = storeRawDataToDataError;
}
+
+ /**
+ * @return the storeRawDataToDataSet
+ */
+ public boolean isStoreRawDataToDataSet() {
+ return storeRawDataToDataSet;
+ }
+
+ /**
+ * @param storeRawDataToDataSet the storeRawDataToDataSet to set
+ */
+ public void setStoreRawDataToDataSet(boolean storeRawDataToDataSet) {
+ this.storeRawDataToDataSet = storeRawDataToDataSet;
+ }
}
Modified: trunk/flatpack/src/main/java/net/sf/flatpack/DataSet.java
===================================================================
--- trunk/flatpack/src/main/java/net/sf/flatpack/DataSet.java 2008-08-31 12:43:32 UTC (rev 401)
+++ trunk/flatpack/src/main/java/net/sf/flatpack/DataSet.java 2008-09-01 13:21:06 UTC (rev 402)
@@ -92,16 +92,16 @@
double getDouble(final String column);
/**
- * Returns the integer value of a specified column
+ * Returns the interger value of a specified column
*
* @param column -
* Name of the column
* @exception NoSuchElementException
* @exception NumberFormatException
- * @return int
+ * @return double
*/
int getInt(final String column);
-
+
/**
* Returns the long value of a specified column
*
@@ -339,4 +339,11 @@
* @return
*/
boolean isRowEmpty();
+
+ /**
+ *
+ * @return the raw data used to create this Row in the DataSet. Parser.setStoreRawDataToDataSet(true)
+ * must be specified before calling this method.
+ */
+ String getRawData();
}
\ No newline at end of file
Modified: trunk/flatpack/src/main/java/net/sf/flatpack/DefaultDataSet.java
===================================================================
--- trunk/flatpack/src/main/java/net/sf/flatpack/DefaultDataSet.java 2008-08-31 12:43:32 UTC (rev 401)
+++ trunk/flatpack/src/main/java/net/sf/flatpack/DefaultDataSet.java 2008-09-01 13:21:06 UTC (rev 402)
@@ -46,6 +46,7 @@
import net.sf.flatpack.structure.ColumnMetaData;
import net.sf.flatpack.structure.Row;
import net.sf.flatpack.util.FPConstants;
+import net.sf.flatpack.util.FPException;
import net.sf.flatpack.util.FPInvalidUsageException;
import net.sf.flatpack.util.FPStringUtils;
import net.sf.flatpack.util.ParserUtils;
@@ -507,7 +508,8 @@
}
/**
- * @throws FPInvalidUsageException
+ * @throws FPInvalidUsageException Parser.isFlagEmptyRows() must be set to true before using this
+ * @throws FPException if cursor is on an invalid row
*/
public boolean isRowEmpty() {
if (!parser.isFlagEmptyRows()) {
@@ -516,6 +518,28 @@
throw new FPInvalidUsageException("Parser.isFlagEmptyRows(true) must be set before using isRowEmpty()");
}
+ if (pointer < 0) {
+ throw new FPException("Cursor on invalid row.. Make sure next() is called and returns true");
+ }
+
return ((Row)rows.get(pointer)).isEmpty();
}
+
+ /**
+ * @throws FPInvalidUsageException
+ * @throws FPException if cursor is on an invalid row
+ */
+ public String getRawData() {
+ if (!parser.isStoreRawDataToDataSet()) {
+ //option needs to be set for this functionality
+ //throw an exception
+ throw new FPInvalidUsageException("Parser.isStoreRawDataToDataSet(true) must be set before using getRawData()");
+ }
+
+ if (pointer < 0) {
+ throw new FPException("Cursor on invalid row.. Make sure next() is called and returns true");
+ }
+
+ return ((Row)rows.get(pointer)).getRawData();
+ }
}
Modified: trunk/flatpack/src/main/java/net/sf/flatpack/Parser.java
===================================================================
--- trunk/flatpack/src/main/java/net/sf/flatpack/Parser.java 2008-08-31 12:43:32 UTC (rev 401)
+++ trunk/flatpack/src/main/java/net/sf/flatpack/Parser.java 2008-09-01 13:21:06 UTC (rev 402)
@@ -153,6 +153,13 @@
void setFlagEmptyRows(final boolean flagEmptyRows);
/**
+ * @return when true, the parser will place the data of the line which failed the parse and
+ * place it into the DataError object. DataError.getRawData() can be called to retrieve
+ * the line.
+ */
+ boolean isStoreRawDataToDataError();
+
+ /**
* when true, the parser will place the data of the line which failed the parse and
* place it into the DataError object. DataError.getRawData() can be called to retrieve
* the line.
@@ -160,4 +167,23 @@
* @param storeRawDataToDataError
*/
void setStoreRawDataToDataError(final boolean storeRawDataToDataError);
+
+
+ /**
+ * @return when true, the parser will place the data of the line into the DataSet object.
+ * DataSet.getRawData() can be called to retrieve the line.
+ */
+ boolean isStoreRawDataToDataSet();
+
+ /**
+ * WARNING!! Setting this option has potential to cause high memory usage.
+ *
+ * when true, the parser will place the data of the line into the DataSet object.
+ * DataSet.getRawData() can be called to retrieve the line.
+ *
+ * @param storeRawDataToDataError
+ */
+ void setStoreRawDataToDataSet(final boolean storeRawDataToDataError);
+
+
}
Modified: trunk/flatpack/src/main/java/net/sf/flatpack/brparse/BuffReaderDelimParser.java
===================================================================
--- trunk/flatpack/src/main/java/net/sf/flatpack/brparse/BuffReaderDelimParser.java 2008-08-31 12:43:32 UTC (rev 401)
+++ trunk/flatpack/src/main/java/net/sf/flatpack/brparse/BuffReaderDelimParser.java 2008-09-01 13:21:06 UTC (rev 402)
@@ -183,6 +183,11 @@
//user has elected to have the parser flag rows that are empty
row.setEmpty(ParserUtils.isListElementsEmpty(columns));
}
+ if (isStoreRawDataToDataSet()) {
+ //user told the parser to keep a copy of the raw data in the row
+ //WARNING potential for high memory usage here
+ row.setRawData(line);
+ }
return row;
}
Modified: trunk/flatpack/src/main/java/net/sf/flatpack/brparse/BuffReaderFixedParser.java
===================================================================
--- trunk/flatpack/src/main/java/net/sf/flatpack/brparse/BuffReaderFixedParser.java 2008-08-31 12:43:32 UTC (rev 401)
+++ trunk/flatpack/src/main/java/net/sf/flatpack/brparse/BuffReaderFixedParser.java 2008-09-01 13:21:06 UTC (rev 402)
@@ -152,6 +152,11 @@
//user has elected to have the parser flag rows that are empty
row.setEmpty(ParserUtils.isListElementsEmpty(row.getCols()));
}
+ if (isStoreRawDataToDataSet()) {
+ //user told the parser to keep a copy of the raw data in the row
+ //WARNING potential for high memory usage here
+ row.setRawData(line);
+ }
return row;
}
Modified: trunk/flatpack/src/main/java/net/sf/flatpack/structure/Row.java
===================================================================
--- trunk/flatpack/src/main/java/net/sf/flatpack/structure/Row.java 2008-08-31 12:43:32 UTC (rev 401)
+++ trunk/flatpack/src/main/java/net/sf/flatpack/structure/Row.java 2008-09-01 13:21:06 UTC (rev 402)
@@ -52,6 +52,8 @@
private String mdkey;
private boolean empty;
+
+ private String rawData;
/**
* Constructs a new Row
@@ -177,4 +179,22 @@
public void setEmpty(boolean empty) {
this.empty = empty;
}
+
+ /**
+ * Raw data used to create the columns for the row
+ *
+ * @return the rawData
+ */
+ public String getRawData() {
+ return rawData;
+ }
+
+ /**
+ * Raw data used to create the columns for the row
+ *
+ * @param rawData the rawData to set
+ */
+ public void setRawData(String rawData) {
+ this.rawData = rawData;
+ }
}
\ No newline at end of file
Modified: trunk/flatpack/src/test/java/net/sf/flatpack/pzparser/PZParserOptsTest.java
===================================================================
--- trunk/flatpack/src/test/java/net/sf/flatpack/pzparser/PZParserOptsTest.java 2008-08-31 12:43:32 UTC (rev 401)
+++ trunk/flatpack/src/test/java/net/sf/flatpack/pzparser/PZParserOptsTest.java 2008-09-01 13:21:06 UTC (rev 402)
@@ -125,6 +125,44 @@
de = (DataError)errors.next();
assertNull("DataError should have <null> line data...", de.getRawData());
}
+
+ public void testStoreRawData() {
+ DataSet ds;
+ final String cols = "column1,column2,column3\r\nVAL1,VAL2,VAL3";
+ Parser p = DefaultParserFactory.getInstance().newDelimitedParser(new StringReader(cols), ',', FPConstants.NO_QUALIFIER);
+ p.setStoreRawDataToDataSet(true);
+ ds = p.parse();
+ ds.next();
+ assertEquals("VAL1,VAL2,VAL3", ds.getRawData());
+
+ p = DefaultParserFactory.getInstance().newDelimitedParser(new StringReader(cols), ',', FPConstants.NO_QUALIFIER);
+ ds = p.parse();
+ ds.next();
+ try {
+ ds.getRawData();
+ fail("Should have received an FPExcpetion...");
+ }catch(FPInvalidUsageException e) {
+ }
+
+ }
+
+ public void testEmptyLastColumn() {
+ //this was reported as a bug in the forums check to see
+ //if we actually have a problem
+ DataSet ds;
+ String cols = "column1,column2,column3\r\nVAL1,VAL2,";
+ Parser p = DefaultParserFactory.getInstance().newDelimitedParser(new StringReader(cols), ',', FPConstants.NO_QUALIFIER);
+ ds = p.parse();
+
+ assertEquals(true, ds.next());
+
+
+ cols = "column1,column2,column3\r\n\"VAL1\",\"VAL2\",\"\"";
+ p = DefaultParserFactory.getInstance().newDelimitedParser(new StringReader(cols), ',', '"');
+ ds = p.parse();
+
+ assertEquals(true, ds.next());
+ }
public static void main(final String[] args) {
junit.textui.TestRunner.run(PZParserOptsTest.class);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
