Menu
▾
▴
flatpack-svn — Subversion commits
You can subscribe to this list here.
| 2006 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
(112) |
Nov
(44) |
Dec
(49) |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 2007 |
Jan
(14) |
Feb
(12) |
Mar
(12) |
Apr
(12) |
May
(6) |
Jun
(11) |
Jul
(10) |
Aug
(6) |
Sep
(17) |
Oct
(3) |
Nov
(22) |
Dec
|
| 2008 |
Jan
(3) |
Feb
(7) |
Mar
(2) |
Apr
|
May
|
Jun
|
Jul
|
Aug
(3) |
Sep
(6) |
Oct
|
Nov
|
Dec
|
| 2009 |
Jan
|
Feb
|
Mar
(1) |
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
|
Nov
|
Dec
|
| 2010 |
Jan
|
Feb
(1) |
Mar
(9) |
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
(4) |
Nov
(2) |
Dec
|
| 2011 |
Jan
|
Feb
|
Mar
(1) |
Apr
|
May
|
Jun
(3) |
Jul
|
Aug
(1) |
Sep
|
Oct
|
Nov
|
Dec
(1) |
| 2012 |
Jan
(3) |
Feb
|
Mar
(2) |
Apr
(8) |
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
|
Nov
|
Dec
|
|
From: <be...@us...> - 2006-10-27 11:53:03
|
Revision: 140
http://svn.sourceforge.net/pzfilereader/?rev=140&view=rev
Author: benoitx
Date: 2006-10-27 04:52:52 -0700 (Fri, 27 Oct 2006)
Log Message:
-----------
Removed some throw Exception
Only non-runtime exceptions should be declared and never at the 'Exception' level, which is far too generic and forces every caller to deal with something which is 'unknonwn'
Modified Paths:
--------------
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/PZParserFactory.java
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/xml/PZMapParser.java
Modified: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/PZParserFactory.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/PZParserFactory.java 2006-10-27 01:05:51 UTC (rev 139)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/PZParserFactory.java 2006-10-27 11:52:52 UTC (rev 140)
@@ -59,7 +59,7 @@
* Name of dataDefinition in the DATAFILE table DATAFILE_DESC
* column
*/
- PZParser newParser(final Connection con, final File dataSource, final String dataDefinition);
+ PZParser newFixedLengthParser(final Connection con, final File dataSource, final String dataDefinition);
/**
* Constructs a new DataSet using the database table file layout method.
@@ -76,7 +76,7 @@
* Name of dataDefinition in the DATAFILE table DATAFILE_DESC
* column
*/
- PZParser newParser(final Connection con, final InputStream dataSourceStream, final String dataDefinition);
+ PZParser newFixedLengthParser(final Connection con, final InputStream dataSourceStream, final String dataDefinition);
/**
* Constructs a new DataSet using the PZMAP XML file layout method. This is
@@ -87,7 +87,7 @@
* @param dataSource -
* Delimited file to read from
*/
- PZParser newParser(final File pzmapXML, final File dataSource);
+ PZParser newFixedLengthParser(final File pzmapXML, final File dataSource);
/**
* New constructor based on InputStream. Constructs a new DataSet using the
@@ -102,7 +102,7 @@
* Delimited file InputStream to read from, user must close them
* after use.
*/
- PZParser newParser(final InputStream pzmapXMLStream, final InputStream dataSourceStream);
+ PZParser newFixedLengthParser(final InputStream pzmapXMLStream, final InputStream dataSourceStream);
//
//
Modified: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java 2006-10-27 01:05:51 UTC (rev 139)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java 2006-10-27 11:52:52 UTC (rev 140)
@@ -581,9 +581,8 @@
*
* @param columnMD
* @return Map
- * @exception Exception
*/
- public static Map calculateRecordLengths(final Map columnMD) throws Exception {
+ public static Map calculateRecordLengths(final Map columnMD) {
final Map recordLengths = new HashMap();
List cmds = null;
@@ -751,19 +750,19 @@
* @param file
* The file.
* @return the InputStream.
- * @throws Exception
+ * @throws FileNotFoundException
*/
- public static InputStream createInputStream(final File file) throws Exception {
+ public static InputStream createInputStream(final File file) throws FileNotFoundException {
if (file == null) {
throw new IllegalArgumentException("null not allowed");
}
- if (!file.exists()) {
- throw new FileNotFoundException("file does not exist " + file.getAbsolutePath());
- }
- if (!file.canRead()) {
- throw new FileNotFoundException("file cannot be read " + file.getAbsolutePath());
- }
- return new FileInputStream(file.getAbsolutePath());
+// if (!file.exists()) {
+// throw new FileNotFoundException("file does not exist " + file.getAbsolutePath());
+// }
+// if (!file.canRead()) {
+// throw new FileNotFoundException("file cannot be read " + file.getAbsolutePath());
+// }
+ return new FileInputStream(file);
}
/**
Modified: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/xml/PZMapParser.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/xml/PZMapParser.java 2006-10-27 01:05:51 UTC (rev 139)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/xml/PZMapParser.java 2006-10-27 11:52:52 UTC (rev 140)
@@ -15,6 +15,7 @@
package net.sf.pzfilereader.xml;
import java.io.File;
+import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Iterator;
@@ -29,6 +30,7 @@
import org.jdom.Attribute;
import org.jdom.Document;
import org.jdom.Element;
+import org.jdom.JDOMException;
import org.jdom.input.SAXBuilder;
/**
@@ -75,9 +77,10 @@
* @param xmlStream
* @return Map
* <records> with their corrisponding
- * @throws Exception
+ * @throws IOException
+ * @throws JDOMException
*/
- public static Map parse(final InputStream xmlStream) throws Exception {
+ public static Map parse(final InputStream xmlStream) throws JDOMException, IOException {
final SAXBuilder builder = new SAXBuilder();
builder.setValidation(true);
// handle the ability to pull DTD from Jar if needed
@@ -110,7 +113,7 @@
// value we are using to mark columns specified outside of a
// <RECORD> element
if (xmlElement.getAttributeValue("id").equals(PZConstants.DETAIL_ID)) {
- throw new Exception("The ID 'detail' on the <RECORD> element is reserved, please select another id");
+ throw new IllegalArgumentException("The ID 'detail' on the <RECORD> element is reserved, please select another id");
}
columns = getColumnChildren(xmlElement);
@@ -145,7 +148,7 @@
}
// helper to retrieve the "COLUMN" elements from the given parent
- private static List getColumnChildren(final Element parent) throws Exception {
+ private static List getColumnChildren(final Element parent) {
final List columnResults = new ArrayList();
final Iterator xmlChildren = parent.getChildren("COLUMN").iterator();
@@ -155,7 +158,7 @@
// make sure the name attribute is present on the column
if (xmlColumn.getAttributeValue("name") == null) {
- throw new Exception("Name attribute is required on the column tag!");
+ throw new IllegalArgumentException("Name attribute is required on the column tag!");
}
cmd.setColName(xmlColumn.getAttributeValue("name"));
@@ -165,7 +168,7 @@
try {
cmd.setColLength(Integer.parseInt(xmlColumn.getAttributeValue("length")));
} catch (final Exception ex) {
- throw new Exception("LENGTH ATTRIBUTE ON COLUMN ELEMENT MUST BE AN INTEGER. GOT: "
+ throw new IllegalArgumentException("LENGTH ATTRIBUTE ON COLUMN ELEMENT MUST BE AN INTEGER. GOT: "
+ xmlColumn.getAttributeValue("length"));
}
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <zep...@us...> - 2006-10-27 01:05:55
|
Revision: 139
http://svn.sourceforge.net/pzfilereader/?rev=139&view=rev
Author: zepernick
Date: 2006-10-26 18:05:51 -0700 (Thu, 26 Oct 2006)
Log Message:
-----------
starting to go through extreme tests
Modified Paths:
--------------
trunk/PZFileReader/src/test/java/net/sf/pzfilereader/parserutils/ParserUtilsSplitLineTest.java
Modified: trunk/PZFileReader/src/test/java/net/sf/pzfilereader/parserutils/ParserUtilsSplitLineTest.java
===================================================================
--- trunk/PZFileReader/src/test/java/net/sf/pzfilereader/parserutils/ParserUtilsSplitLineTest.java 2006-10-27 01:05:18 UTC (rev 138)
+++ trunk/PZFileReader/src/test/java/net/sf/pzfilereader/parserutils/ParserUtilsSplitLineTest.java 2006-10-27 01:05:51 UTC (rev 139)
@@ -106,9 +106,20 @@
check(" a,b,c", ',', '\"', new String[] { "a","b","c" });
// what would you expect of these ones?
- check("a\",b,c", ',', '\"', new String[] { "a", "b", "c" });
+
+ //+++++The parser allows qualified and unqualified elements to be contained
+ //on the same line. so it should break the elements down like so
+ //1 = a" -->" is part of the data since the element did not start with a qualifier
+ //2 = b
+ //3 = c" --> same as #1
+ check("a\",b,c\"", ',', '\"', new String[] { "a\"", "b", "c\"" });
+
+
+
check("\" a,b,c\"", ',', '\"', new String[] { "a,b,c" });
- check(" a, b ,c ", ',', '\"', new String[] { "a","b","c" });
+ //check(" a, b ,c ", ',', '\"', new String[] { "a","b","c" });
+ //++++++I think this should probably generate this
+ check(" a, b ,c ", ',', '\"', new String[] { "a, b ,c" });
// Paul... please put some more whacky stuff here...
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <zep...@us...> - 2006-10-27 01:05:24
|
Revision: 138
http://svn.sourceforge.net/pzfilereader/?rev=138&view=rev
Author: zepernick
Date: 2006-10-26 18:05:18 -0700 (Thu, 26 Oct 2006)
Log Message:
-----------
removed system out
Modified Paths:
--------------
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java
Modified: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java 2006-10-27 01:04:55 UTC (rev 137)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java 2006-10-27 01:05:18 UTC (rev 138)
@@ -157,7 +157,7 @@
//only if the last element was truly qualified
if (beginQualifier && qualifier > 0 && trimmed.length() > 0) {
if (trimmed.charAt(trimmed.length() - 1) == qualifier) {
- System.out.println(">>>>>>>Triming Off Qualifier");
+ // System.out.println(">>>>>>>Triming Off Qualifier");
final String s = trimmed.substring(0, trimmed.length() - 1);
sb.delete(0, sb.length());
sb.append(s);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <zep...@us...> - 2006-10-27 01:05:00
|
Revision: 137
http://svn.sourceforge.net/pzfilereader/?rev=137&view=rev
Author: zepernick
Date: 2006-10-26 18:04:55 -0700 (Thu, 26 Oct 2006)
Log Message:
-----------
fixed bug, should not trim off qualifier unless the element
began with a qulifier
Modified Paths:
--------------
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java
Modified: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java 2006-10-27 00:25:58 UTC (rev 136)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java 2006-10-27 01:04:55 UTC (rev 137)
@@ -154,8 +154,10 @@
// + this needs to be revisited...
final String trimmed = sb.toString().trim();
// remove the ending text qualifier if needed
- if (qualifier > 0 && trimmed.length() > 0) {
+ //only if the last element was truly qualified
+ if (beginQualifier && qualifier > 0 && trimmed.length() > 0) {
if (trimmed.charAt(trimmed.length() - 1) == qualifier) {
+ System.out.println(">>>>>>>Triming Off Qualifier");
final String s = trimmed.substring(0, trimmed.length() - 1);
sb.delete(0, sb.length());
sb.append(s);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <zep...@us...> - 2006-10-27 00:26:05
|
Revision: 136
http://svn.sourceforge.net/pzfilereader/?rev=136&view=rev
Author: zepernick
Date: 2006-10-26 17:25:58 -0700 (Thu, 26 Oct 2006)
Log Message:
-----------
ParserUtils for fixed width files
Added Paths:
-----------
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/FixedWidthParserUtils.java
Added: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/FixedWidthParserUtils.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/FixedWidthParserUtils.java (rev 0)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/FixedWidthParserUtils.java 2006-10-27 00:25:58 UTC (rev 136)
@@ -0,0 +1,122 @@
+/*
+ * ObjectLab, http://www.objectlab.co.uk/open is supporting PZFileReader.
+ *
+ * Based in London, we are world leaders in the design and development
+ * of bespoke applications for the securities financing markets.
+ *
+ * <a href="http://www.objectlab.co.uk/open">Click here to learn more</a>
+ * ___ _ _ _ _ _
+ * / _ \| |__ (_) ___ ___| |_| | __ _| |__
+ * | | | | '_ \| |/ _ \/ __| __| | / _` | '_ \
+ * | |_| | |_) | | __/ (__| |_| |__| (_| | |_) |
+ * \___/|_.__// |\___|\___|\__|_____\__,_|_.__/
+ * |__/
+ *
+ * www.ObjectLab.co.uk
+ *
+ * $Id: ColorProvider.java 74 2006-10-24 22:19:05Z benoitx $
+ *
+ * Copyright 2006 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package net.sf.pzfilereader.util;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import net.sf.pzfilereader.structure.ColumnMetaData;
+import net.sf.pzfilereader.xml.XMLRecordElement;
+
+/**
+ * Collection of parser utilities related to the parsing of
+ * fixed width files.
+ *
+ * @author paul zepernick
+ */
+public class FixedWidthParserUtils {
+ private FixedWidthParserUtils(){
+ }
+
+ /**
+ * Splits up a fixed width line of text
+ *
+ * @param columnMetaData
+ * Collection of ColumnMetaData to parse the line
+ * @param lineToParse
+ * Line of text to be parsed against the ColumnMetaData
+ * @return List
+ * Collection of Strings. Each element representing a column
+ */
+ public static List splitFixedText(final List columnMetaData, final String lineToParse){
+ final List splitResult = new ArrayList();
+ int recPosition = 1;
+ for (int i = 0; i < columnMetaData.size(); i++) {
+ final ColumnMetaData colMetaDataObj = (ColumnMetaData)columnMetaData.get(i);
+ final String tempValue = lineToParse.substring(recPosition - 1, recPosition
+ + colMetaDataObj.getColLength() - 1);
+ recPosition += colMetaDataObj.getColLength();
+ splitResult.add(tempValue.trim());
+ }
+
+ return splitResult;
+ }
+
+ /**
+ * Returns the key to the list of ColumnMetaData objects. Returns the
+ * correct MetaData per the mapping file and the data contained on the line
+ *
+ *
+ * @param columnMD
+ * @param line
+ * @return List - ColumMetaData
+ */
+ public static String getCMDKey(final Map columnMD, final String line) {
+ if (columnMD.size() == 1) {
+ // no <RECORD> elments were specifed for this parse, just return the
+ // detail id
+ return PZConstants.DETAIL_ID;
+ }
+ final Iterator keys = columnMD.keySet().iterator();
+ // loop through the XMLRecordElement objects and see if we need a
+ // different MD object
+ while (keys.hasNext()) {
+ final String key = (String) keys.next();
+ if (key.equals(PZConstants.DETAIL_ID) || key.equals(PZConstants.COL_IDX)) {
+ continue; // skip this key will be assumed if none of the
+ // others match
+ }
+ final XMLRecordElement recordXMLElement = (XMLRecordElement) columnMD.get(key);
+
+ if (recordXMLElement.getEndPositition() > line.length()) {
+ // make sure our substring is not going to fail
+ continue;
+ }
+ final int subfrm = recordXMLElement.getStartPosition() - 1; // convert
+ // to 0
+ // based
+ final int subto = recordXMLElement.getEndPositition();
+ if (line.substring(subfrm, subto).equals(recordXMLElement.getIndicator())) {
+ // we found the MD object we want to return
+ return key;
+ }
+
+ }
+
+ // must be a detail line
+ return PZConstants.DETAIL_ID;
+
+ }
+}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <zep...@us...> - 2006-10-27 00:23:29
|
Revision: 135
http://svn.sourceforge.net/pzfilereader/?rev=135&view=rev
Author: zepernick
Date: 2006-10-26 17:23:25 -0700 (Thu, 26 Oct 2006)
Log Message:
-----------
moved parse to FixedWidthParserUtils
Modified Paths:
--------------
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java
Modified: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java 2006-10-27 00:22:50 UTC (rev 134)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java 2006-10-27 00:23:25 UTC (rev 135)
@@ -37,6 +37,7 @@
import net.sf.pzfilereader.structure.ColumnMetaData;
import net.sf.pzfilereader.structure.Row;
import net.sf.pzfilereader.util.ExcelTransformer;
+import net.sf.pzfilereader.util.FixedWidthParserUtils;
import net.sf.pzfilereader.util.PZConstants;
import net.sf.pzfilereader.util.ParserUtils;
import net.sf.pzfilereader.xml.PZMapParser;
@@ -653,7 +654,7 @@
continue;
}
- final String mdkey = ParserUtils.getCMDKeyForFixedLengthFile(columnMD, line);
+ final String mdkey = FixedWidthParserUtils.getCMDKey(columnMD, line);
final int recordLength = ((Integer) recordLengths.get(mdkey)).intValue();
// Incorrect record length on line log the error. Line will not
@@ -676,19 +677,20 @@
}
}
- int recPosition = 1;
+ //int recPosition = 1;
final Row row = new Row();
row.setMdkey(mdkey.equals(PZConstants.DETAIL_ID) ? null : mdkey); // try
final List cmds = ParserUtils.getColumnMetaData(mdkey, columnMD);
+ row.addColumn(FixedWidthParserUtils.splitFixedText(cmds, line));
// to limit the memory use
// Build the columns for the row
- for (int i = 0; i < cmds.size(); i++) {
- final String tempValue = line.substring(recPosition - 1, recPosition
- + (((ColumnMetaData) cmds.get(i)).getColLength() - 1));
- recPosition += ((ColumnMetaData) cmds.get(i)).getColLength();
- row.addColumn(tempValue.trim());
- }
+ //for (int i = 0; i < cmds.size(); i++) {
+ // final String tempValue = line.substring(recPosition - 1, recPosition
+ // + (((ColumnMetaData) cmds.get(i)).getColLength() - 1));
+ // recPosition += ((ColumnMetaData) cmds.get(i)).getColLength();
+ // row.addColumn(tempValue.trim());
+ // }
row.setRowNumber(lineCount);
// add the row to the array
rows.add(row);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <zep...@us...> - 2006-10-27 00:22:55
|
Revision: 134
http://svn.sourceforge.net/pzfilereader/?rev=134&view=rev
Author: zepernick
Date: 2006-10-26 17:22:50 -0700 (Thu, 26 Oct 2006)
Log Message:
-----------
added a new method to add a collection of columns to the row
Modified Paths:
--------------
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/structure/Row.java
Modified: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/structure/Row.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/structure/Row.java 2006-10-27 00:21:56 UTC (rev 133)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/structure/Row.java 2006-10-27 00:22:50 UTC (rev 134)
@@ -46,6 +46,20 @@
}
cols.add(colValue);
}
+
+ /**
+ * Appends the List of Strings to the existing columns
+ * in the row
+ *
+ * @param columns -
+ * List of Strings to append to the row
+ */
+ public void addColumn(final List columns) {
+ if (cols == null) {
+ cols = new ArrayList();
+ }
+ cols.addAll(columns);
+ }
/**
* Returns the value of a column for a specified column name
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <zep...@us...> - 2006-10-27 00:22:00
|
Revision: 133
http://svn.sourceforge.net/pzfilereader/?rev=133&view=rev
Author: zepernick
Date: 2006-10-26 17:21:56 -0700 (Thu, 26 Oct 2006)
Log Message:
-----------
- fixed line count bug
- moved parse to FixedWidthParserUtils
- moved constants
Modified Paths:
--------------
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/LargeDataSet.java
Modified: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/LargeDataSet.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/LargeDataSet.java 2006-10-27 00:20:31 UTC (rev 132)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/LargeDataSet.java 2006-10-27 00:21:56 UTC (rev 133)
@@ -25,6 +25,8 @@
import net.sf.pzfilereader.ordering.OrderBy;
import net.sf.pzfilereader.structure.ColumnMetaData;
import net.sf.pzfilereader.structure.Row;
+import net.sf.pzfilereader.util.FixedWidthParserUtils;
+import net.sf.pzfilereader.util.PZConstants;
import net.sf.pzfilereader.util.ParserUtils;
import net.sf.pzfilereader.xml.PZMapParser;
@@ -36,11 +38,6 @@
* goTop(), goBottom(), remove(), getIndex(), absolute(), orderRows()
*/
public class LargeDataSet extends DataSet {
-
- private static final String DELIMITED_FILE = "delimited";
-
- private static final String FIXEDLENGTH_FILE = "fixed";
-
private String fileType; // file type being parsed
private BufferedReader br = null; // reader used to read the file
@@ -132,7 +129,7 @@
public LargeDataSet(final InputStream pzmapXMLStream, final InputStream dataSourceStream, final char delimiter,
final char qualifier, final boolean ignoreFirstRecord, final boolean handleShortLines) throws Exception {
- this.fileType = DELIMITED_FILE;
+ this.fileType = PZConstants.DELIMITED_FILE;
this.is = dataSourceStream;
this.isr = new InputStreamReader(is);
this.br = new BufferedReader(this.isr);
@@ -254,7 +251,7 @@
public LargeDataSet(final InputStream dataSource, final char delimiter, final char qualifier, final boolean handleShortLines)
throws Exception {
- this.fileType = DELIMITED_FILE;
+ this.fileType = PZConstants.DELIMITED_FILE;
setHandleShortLines(handleShortLines);
this.is = dataSource;
this.isr = new InputStreamReader(is);
@@ -280,7 +277,7 @@
*/
public LargeDataSet(final InputStream pzmapXMLStream, final InputStream dataSourceStream, final boolean handleShortLines)
throws Exception {
- this.fileType = FIXEDLENGTH_FILE;
+ this.fileType = PZConstants.FIXEDLENGTH_FILE;
this.is = dataSourceStream;
this.isr = new InputStreamReader(is);
this.br = new BufferedReader(this.isr);
@@ -297,7 +294,7 @@
*/
public boolean next() {
try {
- if (this.fileType.equals(DELIMITED_FILE)) {
+ if (this.fileType.equals(PZConstants.DELIMITED_FILE)) {
return readNextDelimited();
}
@@ -408,7 +405,7 @@
* also close out the readers used to read the file in.
*/
public void freeMemory() {
- super.freeMemory();
+ //super.freeMemory();
ParserUtils.closeReader(br);
ParserUtils.closeReader(isr);
@@ -572,8 +569,13 @@
private boolean readNextFixedLen() throws Exception {
String line = null;
- final int aLineCount = 0; // +++++++++++++++++++++++++++++++++ Paul
- // this does not seem incremented at all...
+ //final int aLineCount = 0; // +++++++++++++++++++++++++++++++++ Paul
+ // this does not seem incremented afaLineCountt all...
+ //********************************************************************
+ //Good point, and it is being used to report line errors. Looks like a bug.
+ //Should be using lineCount
+ //PZ
+
boolean readRecordOk = false;
if (getRows() == null) {
@@ -596,7 +598,7 @@
continue;
}
- final String mdkey = ParserUtils.getCMDKeyForFixedLengthFile(getColumnMD(), line);
+ final String mdkey = FixedWidthParserUtils.getCMDKey(getColumnMD(), line);
final int recordLength = ((Integer) recordLengths.get(mdkey)).intValue();
final List cmds = ParserUtils.getColumnMetaData(mdkey, getColumnMD());
@@ -604,7 +606,7 @@
// included in the
// dataset
if (line.length() > recordLength) {
- addError("LINE TOO LONG. LINE IS " + line.length() + " LONG. SHOULD BE " + recordLength, aLineCount, 2);
+ addError("LINE TOO LONG. LINE IS " + line.length() + " LONG. SHOULD BE " + recordLength, this.lineCount, 2);
continue;
} else if (line.length() < recordLength) {
if (isHandleShortLines()) {
@@ -612,25 +614,27 @@
line += ParserUtils.padding(recordLength - line.length(), ' ');
// log a warning
- addError("PADDED LINE TO CORRECT RECORD LENGTH", aLineCount, 1);
+ addError("PADDED LINE TO CORRECT RECORD LENGTH", this.lineCount, 1);
} else {
- addError("LINE TOO SHORT. LINE IS " + line.length() + " LONG. SHOULD BE " + recordLength, aLineCount, 2);
+ addError("LINE TOO SHORT. LINE IS " + line.length() + " LONG. SHOULD BE " + recordLength, this.lineCount, 2);
continue;
}
}
- int recPosition = 1;
+ //int recPosition = 1;
final Row row = new Row();
row.setMdkey(mdkey.equals("detail") ? null : mdkey); // try to
// limit the memory use
+
// Build the columns for the row
- for (int i = 0; i < cmds.size(); i++) {
- final String tempValue = line.substring(recPosition - 1, recPosition
- + (((ColumnMetaData) cmds.get(i)).getColLength() - 1));
- recPosition += ((ColumnMetaData) cmds.get(i)).getColLength();
- row.addColumn(tempValue.trim());
- }
- row.setRowNumber(aLineCount);
+ row.addColumn(FixedWidthParserUtils.splitFixedText(cmds, line));
+ //for (int i = 0; i < cmds.size(); i++) {
+ // final String tempValue = line.substring(recPosition - 1, recPosition
+ // + (((ColumnMetaData) cmds.get(i)).getColLength() - 1));
+ // recPosition += ((ColumnMetaData) cmds.get(i)).getColLength();
+ // row.addColumn(tempValue.trim());
+ // }
+ row.setRowNumber(this.lineCount);
// add the row to the array
getRows().add(row);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <zep...@us...> - 2006-10-27 00:20:37
|
Revision: 132
http://svn.sourceforge.net/pzfilereader/?rev=132&view=rev
Author: zepernick
Date: 2006-10-26 17:20:31 -0700 (Thu, 26 Oct 2006)
Log Message:
-----------
moved fixed width method to FixedWidthParserUtils
deprecated method
Modified Paths:
--------------
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java
Modified: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java 2006-10-27 00:07:34 UTC (rev 131)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java 2006-10-27 00:20:31 UTC (rev 132)
@@ -615,6 +615,8 @@
* @param columnMD
* @param line
* @return List - ColumMetaData
+ * @deprecated Moved to FixedWidthParserUtils.getCMDKey()
+ *
*/
public static String getCMDKeyForFixedLengthFile(final Map columnMD, final String line) {
if (columnMD.size() == 1) {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <zep...@us...> - 2006-10-27 00:07:40
|
Revision: 131
http://svn.sourceforge.net/pzfilereader/?rev=131&view=rev
Author: zepernick
Date: 2006-10-26 17:07:34 -0700 (Thu, 26 Oct 2006)
Log Message:
-----------
moved some constants in from LargeDataSet
Modified Paths:
--------------
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/PZConstants.java
Modified: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/PZConstants.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/PZConstants.java 2006-10-27 00:06:59 UTC (rev 130)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/PZConstants.java 2006-10-27 00:07:34 UTC (rev 131)
@@ -1,5 +1,34 @@
-/**
+/*
+ * ObjectLab, http://www.objectlab.co.uk/open is supporting PZFileReader.
*
+ * Based in London, we are world leaders in the design and development
+ * of bespoke applications for the securities financing markets.
+ *
+ * <a href="http://www.objectlab.co.uk/open">Click here to learn more</a>
+ * ___ _ _ _ _ _
+ * / _ \| |__ (_) ___ ___| |_| | __ _| |__
+ * | | | | '_ \| |/ _ \/ __| __| | / _` | '_ \
+ * | |_| | |_) | | __/ (__| |_| |__| (_| | |_) |
+ * \___/|_.__// |\___|\___|\__|_____\__,_|_.__/
+ * |__/
+ *
+ * www.ObjectLab.co.uk
+ *
+ * $Id: ColorProvider.java 74 2006-10-24 22:19:05Z benoitx $
+ *
+ * Copyright 2006 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
*/
package net.sf.pzfilereader.util;
@@ -11,7 +40,11 @@
public static final String DETAIL_ID = "detail";
public static final String COL_IDX = "colIndex";
+
+ public static final String DELIMITED_FILE = "delimited";
+ public static final String FIXEDLENGTH_FILE = "fixed";
+
private PZConstants() {
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <zep...@us...> - 2006-10-27 00:07:07
|
Revision: 130
http://svn.sourceforge.net/pzfilereader/?rev=130&view=rev
Author: zepernick
Date: 2006-10-26 17:06:59 -0700 (Thu, 26 Oct 2006)
Log Message:
-----------
added a better comment to getColumns()
Modified Paths:
--------------
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/xml/XMLRecordElement.java
Modified: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/xml/XMLRecordElement.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/xml/XMLRecordElement.java 2006-10-26 15:54:43 UTC (rev 129)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/xml/XMLRecordElement.java 2006-10-27 00:06:59 UTC (rev 130)
@@ -98,7 +98,9 @@
}
/**
- * @return Returns the columns.
+ * @return List
+ * Collection of ColumnMetaData objects
+ * @see net.sf.pzfilereader.structure.ColumnMetaData
*/
public List getColumns() {
return columns;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <be...@us...> - 2006-10-26 15:54:46
|
Revision: 129
http://svn.sourceforge.net/pzfilereader/?rev=129&view=rev
Author: benoitx
Date: 2006-10-26 08:54:43 -0700 (Thu, 26 Oct 2006)
Log Message:
-----------
Added a heuristic test that proves that using a StringBuffer delete is better than creating a new one...
Modified Paths:
--------------
trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/Examples.java
Modified: trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/Examples.java
===================================================================
--- trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/Examples.java 2006-10-26 15:53:08 UTC (rev 128)
+++ trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/Examples.java 2006-10-26 15:54:43 UTC (rev 129)
@@ -68,7 +68,8 @@
menu.addMenuItem("DelimitedMultiLine", "doDelimitedMultiLine", false);
menu.addMenuItem("NumericsAndDates", "doNumericsAndDates", false);
menu.addMenuItem("Ask for GC", "doGC", false);
-
+ menu.addMenuItem("Test StringBuffer", "doStringBuffer", false);
+
menu.addMenuItem("Who you gonna call?", "doCall", false);
menu.displayMenu();
}
@@ -82,69 +83,69 @@
public void doCSVHeaderAndTrailer() {
try {
final String mapping = ConsoleMenu.getString("Mapping ", CSVHeaderAndTrailer.getDefaultMapping());
- final String data = ConsoleMenu.getString( "Data ", CSVHeaderAndTrailer.getDefaultDataFile());
- CSVHeaderAndTrailer.call(mapping,data);
+ final String data = ConsoleMenu.getString("Data ", CSVHeaderAndTrailer.getDefaultDataFile());
+ CSVHeaderAndTrailer.call(mapping, data);
} catch (Exception e) {
e.printStackTrace();
}
}
-
+
public void doCSVPerformanceTest() {
try {
final String mapping = ConsoleMenu.getString("CSV File ", "SampleCSV.csv");
final boolean data = ConsoleMenu.getBoolean("Traverse the entire parsed file", true);
final boolean verbose = ConsoleMenu.getBoolean("Verbose", false);
- CSVPerformanceTest.call(mapping,verbose,data);
+ CSVPerformanceTest.call(mapping, verbose, data);
} catch (Exception e) {
e.printStackTrace();
}
}
-
+
public void doDelimitedWithPZMap() {
try {
final String mapping = ConsoleMenu.getString("Mapping ", DelimitedWithPZMap.getDefaultMapping());
- final String data = ConsoleMenu.getString( "Data ", DelimitedWithPZMap.getDefaultDataFile());
- DelimitedWithPZMap.call(mapping,data);
+ final String data = ConsoleMenu.getString("Data ", DelimitedWithPZMap.getDefaultDataFile());
+ DelimitedWithPZMap.call(mapping, data);
} catch (Exception e) {
e.printStackTrace();
}
}
-
+
public void doDelimitedWithPZMapErrors() {
try {
final String mapping = ConsoleMenu.getString("Mapping ", DelimitedWithPZMapErrors.getDefaultMapping());
- final String data = ConsoleMenu.getString( "Data ", DelimitedWithPZMapErrors.getDefaultDataFile());
- DelimitedWithPZMapErrors.call(mapping,data);
+ final String data = ConsoleMenu.getString("Data ", DelimitedWithPZMapErrors.getDefaultDataFile());
+ DelimitedWithPZMapErrors.call(mapping, data);
} catch (Exception e) {
e.printStackTrace();
}
}
-
+
public void doDelimitedFileExportToExcel() {
try {
final String mapping = ConsoleMenu.getString("Mapping ", DelimitedFileExportToExcel.getDefaultMapping());
- final String data = ConsoleMenu.getString( "Data ", DelimitedFileExportToExcel.getDefaultDataFile());
- DelimitedFileExportToExcel.call(mapping,data);
+ final String data = ConsoleMenu.getString("Data ", DelimitedFileExportToExcel.getDefaultDataFile());
+ DelimitedFileExportToExcel.call(mapping, data);
} catch (Exception e) {
e.printStackTrace();
}
}
-
+
public void doFixedLengthWithPZMap() {
try {
final String mapping = ConsoleMenu.getString("Mapping ", FixedLengthWithPZMap.getDefaultMapping());
- final String data = ConsoleMenu.getString( "Data ", FixedLengthWithPZMap.getDefaultDataFile());
- FixedLengthWithPZMap.call(mapping,data);
+ final String data = ConsoleMenu.getString("Data ", FixedLengthWithPZMap.getDefaultDataFile());
+ FixedLengthWithPZMap.call(mapping, data);
} catch (Exception e) {
e.printStackTrace();
}
}
-
+
public void doFixedLengthHeaderAndTrailer() {
try {
final String mapping = ConsoleMenu.getString("Mapping ", FixedLengthHeaderAndTrailer.getDefaultMapping());
- final String data = ConsoleMenu.getString( "Data ", FixedLengthHeaderAndTrailer.getDefaultDataFile());
- FixedLengthHeaderAndTrailer.call(mapping,data);
+ final String data = ConsoleMenu.getString("Data ", FixedLengthHeaderAndTrailer.getDefaultDataFile());
+ FixedLengthHeaderAndTrailer.call(mapping, data);
} catch (Exception e) {
e.printStackTrace();
}
@@ -153,19 +154,18 @@
public void doLargeDelimitedWithPZMap() {
try {
final String mapping = ConsoleMenu.getString("Mapping ", LargeDelimitedWithPZMap.getDefaultMapping());
- final String data = ConsoleMenu.getString( "Data ", LargeDelimitedWithPZMap.getDefaultDataFile());
- LargeDelimitedWithPZMap.call(mapping,data);
+ final String data = ConsoleMenu.getString("Data ", LargeDelimitedWithPZMap.getDefaultDataFile());
+ LargeDelimitedWithPZMap.call(mapping, data);
} catch (Exception e) {
e.printStackTrace();
}
}
-
public void doLargeFixedLengthWithPZMap() {
try {
final String mapping = ConsoleMenu.getString("Mapping ", LargeFixedLengthWithPZMap.getDefaultMapping());
- final String data = ConsoleMenu.getString( "Data ", LargeFixedLengthWithPZMap.getDefaultDataFile());
- LargeFixedLengthWithPZMap.call(mapping,data);
+ final String data = ConsoleMenu.getString("Data ", LargeFixedLengthWithPZMap.getDefaultDataFile());
+ LargeFixedLengthWithPZMap.call(mapping, data);
} catch (Exception e) {
e.printStackTrace();
}
@@ -177,7 +177,7 @@
final int rows = ConsoleMenu.getInt("Number of rows", 2000000);
String filename = "LargeSampleCSV.csv";
CSVTestFileCreator.createFile(cols, rows, filename);
-
+
System.out.println("Large file created");
CSVLarge.call(filename);
@@ -185,10 +185,10 @@
e.printStackTrace();
}
}
-
+
public void doLowLevelParse() {
try {
- final String data = ConsoleMenu.getString( "Data ", LowLevelParse.getDefaultDataFile());
+ final String data = ConsoleMenu.getString("Data ", LowLevelParse.getDefaultDataFile());
LowLevelParse.call(data);
} catch (Exception e) {
e.printStackTrace();
@@ -197,7 +197,7 @@
public void doDelimitedMultiLine() {
try {
- final String data = ConsoleMenu.getString( "Data ", DelimitedMultiLine.getDefaultDataFile());
+ final String data = ConsoleMenu.getString("Data ", DelimitedMultiLine.getDefaultDataFile());
DelimitedMultiLine.call(data);
} catch (Exception e) {
e.printStackTrace();
@@ -207,8 +207,8 @@
public void doNumericsAndDates() {
try {
final String mapping = ConsoleMenu.getString("Mapping ", NumericsAndDates.getDefaultMapping());
- final String data = ConsoleMenu.getString( "Data ", NumericsAndDates.getDefaultDataFile());
- NumericsAndDates.call(mapping,data);
+ final String data = ConsoleMenu.getString("Data ", NumericsAndDates.getDefaultDataFile());
+ NumericsAndDates.call(mapping, data);
} catch (Exception e) {
e.printStackTrace();
}
@@ -218,4 +218,33 @@
System.gc();
}
+ public void doStringBuffer() {
+ final int repeat = ConsoleMenu.getInt("How many times?", 100000);
+ final int characters = ConsoleMenu.getInt("How many char?", 20);
+
+ long start = System.currentTimeMillis();
+ for (int i = 0; i < repeat; i++) {
+ StringBuffer sb = new StringBuffer();
+ for (int u = 0; u < 1000; u++) {
+ sb.append("h");
+ }
+ }
+ long stop = System.currentTimeMillis();
+
+ System.out.println("Creating new SB " + (stop - start) + " ms.");
+
+ start = System.currentTimeMillis();
+ StringBuffer sb = new StringBuffer();
+ for (int i = 0; i < repeat; i++) {
+ for (int u = 0; u < characters; u++) {
+ sb.append("h");
+ }
+ sb.delete(0, sb.length());
+ }
+ stop = System.currentTimeMillis();
+
+ System.out.println("Deleting existing SB " + (stop - start) + " ms.");
+
+ }
+
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <be...@us...> - 2006-10-26 15:53:11
|
Revision: 128
http://svn.sourceforge.net/pzfilereader/?rev=128&view=rev
Author: benoitx
Date: 2006-10-26 08:53:08 -0700 (Thu, 26 Oct 2006)
Log Message:
-----------
Try to reduce the number of trimmings but Paul, could you check the comments //+ as I believe that those tests are redudant...
Modified Paths:
--------------
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java
Modified: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java 2006-10-26 15:51:35 UTC (rev 127)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java 2006-10-26 15:53:08 UTC (rev 128)
@@ -740,7 +740,8 @@
while ((line = br.readLine()) != null) {
lineCount++;
/** empty line skip past it */
- if (!processingMultiLine && line.trim().length() == 0) {
+ String trimmed = line.trim();
+ if (!processingMultiLine && trimmed.length() == 0) {
continue;
}
@@ -759,7 +760,7 @@
// any line breaks in the middle of the record, this will only
// be checked if we have specified a delimiter
// ********************************************************
- final char[] chrArry = line.trim().toCharArray();
+ final char[] chrArry = trimmed.toCharArray();
if (!processingMultiLine && delimiter > 0) {
processingMultiLine = ParserUtils.isMultiLine(chrArry, delimiter, qualifier);
}
@@ -767,26 +768,25 @@
// check to see if we have reached the end of the linebreak in
// the record
- if (processingMultiLine && lineData.trim().length() > 0) {
+ final String trimmedLineData = lineData.trim();
+ if (processingMultiLine && trimmedLineData.length() > 0) {
// need to do one last check here. it is possible that the "
// could be part of the data
// excel will escape these with another quote; here is some
// data "" This would indicate
// there is more to the multiline
- String trimmed = line.trim();
if (trimmed.charAt(trimmed.length() - 1) == qualifier && !trimmed.endsWith("" + qualifier + qualifier)) {
// it is safe to assume we have reached the end of the
// line break
processingMultiLine = false;
- if (lineData.trim().length() > 0) {
+ if (trimmedLineData.length() > 0) { //+ would always be true surely....
lineData += "\r\n";
}
lineData += line;
} else {
-
// check to see if this is the last line of the record
// looking for a qualifier followed by a delimiter
- if (lineData.trim().length() > 0) {
+ if (trimmedLineData.length() > 0) { //+ here again, this should always be true...
lineData += "\r\n";
}
lineData += line;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <be...@us...> - 2006-10-26 15:51:42
|
Revision: 127
http://svn.sourceforge.net/pzfilereader/?rev=127&view=rev
Author: benoitx
Date: 2006-10-26 08:51:35 -0700 (Thu, 26 Oct 2006)
Log Message:
-----------
try to reduce memory requirements by trimming to size the list.
Modified Paths:
--------------
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java
Modified: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java 2006-10-26 15:50:17 UTC (rev 126)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java 2006-10-26 15:51:35 UTC (rev 127)
@@ -84,7 +84,7 @@
// line which has not yet been read
// check to see if there is a text qualifier
final char currentChar = line.charAt(i);
- final String currentString = String.valueOf(currentChar);
+// final String currentString = String.valueOf(currentChar);
if (qualifier > 0) {
if (currentChar == qualifier && !beginQualifier && !beginNoQualifier) {
// begining of a set of data
@@ -152,7 +152,7 @@
}
// + this needs to be revisited...
- String trimmed = sb.toString().trim();
+ final String trimmed = sb.toString().trim();
// remove the ending text qualifier if needed
if (qualifier > 0 && trimmed.length() > 0) {
if (trimmed.charAt(trimmed.length() - 1) == qualifier) {
@@ -162,7 +162,7 @@
}
}
- String trimmed2 = line.trim();
+ final String trimmed2 = line.trim();
int lengthLeft = trimmed2.length();
if (qualifier <= 0 || beginQualifier || beginNoQualifier || lengthLeft > 0
&& trimmed2.charAt(lengthLeft - 1) == delimiter) {
@@ -176,6 +176,8 @@
sb = null;
+ list.trimToSize();
+
return list;
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <be...@us...> - 2006-10-26 15:50:23
|
Revision: 126
http://svn.sourceforge.net/pzfilereader/?rev=126&view=rev
Author: benoitx
Date: 2006-10-26 08:50:17 -0700 (Thu, 26 Oct 2006)
Log Message:
-----------
Added a couple of whacky tests, some fail (on purpose); Paul could you check what results you expect and create a few more?
Thanks
Modified Paths:
--------------
trunk/PZFileReader/src/test/java/net/sf/pzfilereader/parserutils/ParserUtilsSplitLineTest.java
Modified: trunk/PZFileReader/src/test/java/net/sf/pzfilereader/parserutils/ParserUtilsSplitLineTest.java
===================================================================
--- trunk/PZFileReader/src/test/java/net/sf/pzfilereader/parserutils/ParserUtilsSplitLineTest.java 2006-10-26 14:04:40 UTC (rev 125)
+++ trunk/PZFileReader/src/test/java/net/sf/pzfilereader/parserutils/ParserUtilsSplitLineTest.java 2006-10-26 15:50:17 UTC (rev 126)
@@ -59,7 +59,7 @@
}
/**
- * Test without any line breaks
+ * Test with any line breaks
*
*/
public void testLineBreaks() {
@@ -70,7 +70,7 @@
final String txtToParse = UnitTestUtils.buildDelimString(DELIMITED_DATA_WITH_BREAKS, d, q);
- final List splitLineResults = ParserUtils.splitLine(txtToParse.toString(), d, q);
+ final List splitLineResults = ParserUtils.splitLine(txtToParse, d, q);
// check to make sure we have the same amount of elements which were
// expected
@@ -82,25 +82,51 @@
assertEquals("Data Element Value Does Not Match (d = " + d + " q = " + q + ")", DELIMITED_DATA_WITH_BREAKS[j],
(String) splitLineResults.get(j));
}
-
}
}
/**
* Test to make sure we get the correct amount of elements for malformed
* data
- *
- * @param args
*/
public void testMalformedData() {
final List splitLineResults = ParserUtils.splitLine(DELIMITED_BAD_DATA, ',', '\"');
assertEquals("Expecting 2 Data Elements From The Malformed Data", 2, splitLineResults.size());
+ }
+ /**
+ * Test some extreme cases
+ */
+ public void testSomeExtremeCases() {
+ check("\"a,b,c\"", ',', '\"', new String[] { "a,b,c" });
+ check("\"a,b\",\"c\"", ',', '\"', new String[] { "a,b", "c" });
+ check("a,b,c", ',', '\"', new String[] { "a", "b", "c" });
+ check(" a,b,c", ',', '\"', new String[] { "a", "b", "c" });
+ check(" a,b,c", ',', '\"', new String[] { "a","b","c" });
+
+ // what would you expect of these ones?
+ check("a\",b,c", ',', '\"', new String[] { "a", "b", "c" });
+ check("\" a,b,c\"", ',', '\"', new String[] { "a,b,c" });
+ check(" a, b ,c ", ',', '\"', new String[] { "a","b","c" });
+
+ // Paul... please put some more whacky stuff here...
+
}
+ private void check(final String txtToParse, final char delim, final char qualifier, final String[] expected) {
+ final List splitLineResults = ParserUtils.splitLine(txtToParse, delim, qualifier);
+
+ assertEquals(
+ "Did Not Get Amount Of Elements Expected (d = " + delim + " q = " + qualifier + ") txt [" + txtToParse + "]",
+ expected.length, splitLineResults.size());
+
+ for (int i = 0; i < expected.length; i++) {
+ assertEquals("expecting...", expected[i], splitLineResults.get(i));
+ }
+ }
+
public static void main(final String[] args) {
junit.textui.TestRunner.run(ParserUtilsSplitLineTest.class);
}
-
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <be...@us...> - 2006-10-26 14:04:53
|
Revision: 125
http://svn.sourceforge.net/pzfilereader/?rev=125&view=rev
Author: benoitx
Date: 2006-10-26 07:04:40 -0700 (Thu, 26 Oct 2006)
Log Message:
-----------
First cut at some interfaces. Paul, could you review and tell me if you think that they are well separated.
I think that PZParserFactory.java and PZParser.java are ok but have I put everything that is required for the manipulation
of a DataSet in IDataSet.java?
Modified Paths:
--------------
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java
Added Paths:
-----------
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/IDataSet.java
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/PZParser.java
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/PZParserFactory.java
Modified: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java 2006-10-26 08:58:08 UTC (rev 124)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java 2006-10-26 14:04:40 UTC (rev 125)
@@ -49,7 +49,7 @@
* @version 2.0.1
* @todo Ought to implement an interface for the access to data.
*/
-public class DataSet {
+public class DataSet implements IDataSet {
/** Array to hold the rows and their values in the text file */
private List rows = null;
@@ -906,27 +906,22 @@
// row.setValue(ParserUtils.findColumn(columnName, cmds), value);
}
- /**
- * Goes to the top of the data set. This will put the pointer one record
- * before the first in the set. Next() will have to be called to get the
- * first record after this call.
+ /* (non-Javadoc)
+ * @see net.sf.pzfilereader.IDataSet#goTop()
*/
public void goTop() {
pointer = -1;
}
- /**
- * Goes to the last record in the dataset
+ /* (non-Javadoc)
+ * @see net.sf.pzfilereader.IDataSet#goBottom()
*/
public void goBottom() {
pointer = rows.size() - 1;
}
- /**
- * Moves to the next record in the set. Returns true if move was a success,
- * false if not
- *
- * @return boolean
+ /* (non-Javadoc)
+ * @see net.sf.pzfilereader.IDataSet#next()
*/
public boolean next() {
if (pointer < rows.size() && pointer + 1 != rows.size()) {
@@ -936,11 +931,8 @@
return false;
}
- /**
- * Moves back to the previous record in the set return true if move was a
- * success, false if not
- *
- * @return boolean
+ /* (non-Javadoc)
+ * @see net.sf.pzfilereader.IDataSet#previous()
*/
public boolean previous() {
if (pointer <= 0) {
@@ -950,13 +942,8 @@
return true;
}
- /**
- * Returns the string value of a specified column
- *
- * @param column -
- * Name of the column
- * @exception NoSuchElementException
- * @return String
+ /* (non-Javadoc)
+ * @see net.sf.pzfilereader.IDataSet#getString(java.lang.String)
*/
public String getString(final String column) {
final Row row = (Row) rows.get(pointer);
@@ -983,14 +970,8 @@
return s;
}
- /**
- * Returns the double value of a specified column
- *
- * @param column -
- * Name of the column
- * @exception NoSuchElementException
- * @exception NumberFormatException
- * @return double
+ /* (non-Javadoc)
+ * @see net.sf.pzfilereader.IDataSet#getDouble(java.lang.String)
*/
public double getDouble(final String column) {
final StringBuffer newString = new StringBuffer();
@@ -1023,14 +1004,8 @@
return Double.parseDouble(newString.toString());
}
- /**
- * Returns the interger value of a specified column
- *
- * @param column -
- * Name of the column
- * @exception NoSuchElementException
- * @exception NumberFormatException
- * @return double
+ /* (non-Javadoc)
+ * @see net.sf.pzfilereader.IDataSet#getInt(java.lang.String)
*/
public int getInt(final String column) {
final StringBuffer newString = new StringBuffer();
@@ -1063,15 +1038,8 @@
return Integer.parseInt(newString.toString());
}
- /**
- * Returns the date value of a specified column. This assumes the date is in
- * yyyyMMdd. If your date is not in this format, see
- * getDate(String,SimpleDateFormat)
- *
- * @param column -
- * Name of the column
- * @exception ParseException
- * @return Date
+ /* (non-Javadoc)
+ * @see net.sf.pzfilereader.IDataSet#getDate(java.lang.String)
*/
public Date getDate(final String column) throws ParseException {
final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
@@ -1084,18 +1052,8 @@
return sdf.parse(s);
}
- /**
- * Returns the date value of a specified column. This should be used if the
- * date is NOT in yyyyMMdd format. The SimpleDateFormat object will specify
- * what kind of format the date is in.
- *
- * @param column -
- * Name of the column
- * @param sdf -
- * SimpleDateFormat of the date
- * @exception ParseException
- * @see java.text.SimpleDateFormat
- * @return Date
+ /* (non-Javadoc)
+ * @see net.sf.pzfilereader.IDataSet#getDate(java.lang.String, java.text.SimpleDateFormat)
*/
public Date getDate(final String column, final SimpleDateFormat sdf) throws ParseException {
final Row row = (Row) rows.get(pointer);
@@ -1107,11 +1065,8 @@
return sdf.parse(s);
}
- /**
- * Returns a String array of column names in the DataSet. This will assume
- * 'detail' <RECORD> ID.
- *
- * @return String[]
+ /* (non-Javadoc)
+ * @see net.sf.pzfilereader.IDataSet#getColumns()
*/
public String[] getColumns() {
ColumnMetaData column = null;
@@ -1130,12 +1085,8 @@
return array;
}
- /**
- * Returns a String array of column names in the DataSet for a given
- * <RECORD> id
- *
- * @param recordID
- * @return String[]
+ /* (non-Javadoc)
+ * @see net.sf.pzfilereader.IDataSet#getColumns(java.lang.String)
*/
public String[] getColumns(final String recordID) {
String[] array = null;
@@ -1152,22 +1103,15 @@
return array;
}
- /**
- * Returns the line number the pointer is on. These are the actual line
- * numbers from the flat file, before any sorting.
- *
- * @exception NoSuchElementException
- * @exception NumberFormatException
- * @return int
+ /* (non-Javadoc)
+ * @see net.sf.pzfilereader.IDataSet#getRowNo()
*/
public int getRowNo() {
return ((Row) rows.get(pointer)).getRowNumber();
}
- /**
- * Returns A Collection Of DataErrors that happened during processing
- *
- * @return Vector
+ /* (non-Javadoc)
+ * @see net.sf.pzfilereader.IDataSet#getErrors()
*/
public List getErrors() {
return errors;
@@ -1192,19 +1136,16 @@
errors.add(de);
}
- /**
- * Removes a row from the dataset. Once the row is removed the pointer will
- * be sitting on the record previous to the deleted row.
+ /* (non-Javadoc)
+ * @see net.sf.pzfilereader.IDataSet#remove()
*/
public void remove() {
rows.remove(pointer);
pointer--;
}
- /**
- * Returns the index the pointer is on for the array
- *
- * @return int
+ /* (non-Javadoc)
+ * @see net.sf.pzfilereader.IDataSet#getIndex()
*/
public int getIndex() {
return pointer;
@@ -1240,21 +1181,15 @@
return rowID.equals(recordID);
}
- /**
- * Returns the total number of rows parsed in from the file
- *
- *
- * @return int - Row Count
+ /* (non-Javadoc)
+ * @see net.sf.pzfilereader.IDataSet#getRowCount()
*/
public int getRowCount() {
return rows.size();
}
- /**
- * Returns total number of records which contained a parse error in the
- * file.
- *
- * @return int - Record Error Count
+ /* (non-Javadoc)
+ * @see net.sf.pzfilereader.IDataSet#getErrorCount()
*/
public int getErrorCount() {
if (getErrors() != null) {
@@ -1264,14 +1199,8 @@
return 0;
}
- /**
- * Returns true or false as to whether or not the line number contains an
- * error. The import will skip the line if it contains an error and it will
- * not be processed
- *
- * @param lineNo -
- * int line number
- * @return boolean
+ /* (non-Javadoc)
+ * @see net.sf.pzfilereader.IDataSet#isAnError(int)
*/
public boolean isAnError(final int lineNo) {
for (int i = 0; i < errors.size(); i++) {
@@ -1282,17 +1211,8 @@
return false;
}
- /**
- * Orders the data by column(s) specified. This will reposition the cursor
- * to the top of the DataSet when executed. This is currently not supported
- * when specying <RECORD> elements in the mapping. An exception will be
- * thrown if this situation occurs
- *
- * @param ob -
- * OrderBy object
- * @exception Exception
- * @see net.sf.pzfilereader.ordering.OrderBy
- * @see net.sf.pzfilereader.ordering.OrderColumn
+ /* (non-Javadoc)
+ * @see net.sf.pzfilereader.IDataSet#orderRows(net.sf.pzfilereader.ordering.OrderBy)
*/
public void orderRows(final OrderBy ob) throws Exception {
// PZ try to handle other <records> by sending them to
@@ -1404,6 +1324,9 @@
this.columnMD = columnMD;
}
+ /* (non-Javadoc)
+ * @see net.sf.pzfilereader.IDataSet#getRows()
+ */
public List getRows() {
return rows;
}
Added: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/IDataSet.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/IDataSet.java (rev 0)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/IDataSet.java 2006-10-26 14:04:40 UTC (rev 125)
@@ -0,0 +1,222 @@
+/*
+ * ObjectLab, http://www.objectlab.co.uk/open is supporting PZFileReader.
+ *
+ * Based in London, we are world leaders in the design and development
+ * of bespoke applications for the securities financing markets.
+ *
+ * <a href="http://www.objectlab.co.uk/open">Click here to learn more</a>
+ * ___ _ _ _ _ _
+ * / _ \| |__ (_) ___ ___| |_| | __ _| |__
+ * | | | | '_ \| |/ _ \/ __| __| | / _` | '_ \
+ * | |_| | |_) | | __/ (__| |_| |__| (_| | |_) |
+ * \___/|_.__// |\___|\___|\__|_____\__,_|_.__/
+ * |__/
+ *
+ * www.ObjectLab.co.uk
+ *
+ * $Id: ColorProvider.java 74 2006-10-24 22:19:05Z benoitx $
+ *
+ * Copyright 2006 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package net.sf.pzfilereader;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import net.sf.pzfilereader.ordering.OrderBy;
+
+public interface IDataSet {
+
+ /**
+ * Goes to the top of the data set. This will put the pointer one record
+ * before the first in the set. Next() will have to be called to get the
+ * first record after this call.
+ */
+ void goTop();
+
+ /**
+ * Goes to the last record in the dataset
+ */
+ void goBottom();
+
+ /**
+ * Moves to the next record in the set. Returns true if move was a success,
+ * false if not
+ *
+ * @return boolean
+ */
+ boolean next();
+
+ /**
+ * Moves back to the previous record in the set return true if move was a
+ * success, false if not
+ *
+ * @return boolean
+ */
+ boolean previous();
+
+ /**
+ * Returns the string value of a specified column
+ *
+ * @param column -
+ * Name of the column
+ * @exception NoSuchElementException
+ * @return String
+ */
+ String getString(final String column);
+
+ /**
+ * Returns the double value of a specified column
+ *
+ * @param column -
+ * Name of the column
+ * @exception NoSuchElementException
+ * @exception NumberFormatException
+ * @return double
+ */
+ double getDouble(final String column);
+
+ /**
+ * Returns the interger value of a specified column
+ *
+ * @param column -
+ * Name of the column
+ * @exception NoSuchElementException
+ * @exception NumberFormatException
+ * @return double
+ */
+ int getInt(final String column);
+
+ /**
+ * Returns the date value of a specified column. This assumes the date is in
+ * yyyyMMdd. If your date is not in this format, see
+ * getDate(String,SimpleDateFormat)
+ *
+ * @param column -
+ * Name of the column
+ * @exception ParseException
+ * @return Date
+ */
+ Date getDate(final String column) throws ParseException;
+
+ /**
+ * Returns the date value of a specified column. This should be used if the
+ * date is NOT in yyyyMMdd format. The SimpleDateFormat object will specify
+ * what kind of format the date is in.
+ *
+ * @param column -
+ * Name of the column
+ * @param sdf -
+ * SimpleDateFormat of the date
+ * @exception ParseException
+ * @see java.text.SimpleDateFormat
+ * @return Date
+ */
+ Date getDate(final String column, final SimpleDateFormat sdf) throws ParseException;
+
+ /**
+ * Returns a String array of column names in the DataSet. This will assume
+ * 'detail' <RECORD> ID.
+ *
+ * @return String[]
+ */
+ String[] getColumns();
+
+ /**
+ * Returns a String array of column names in the DataSet for a given
+ * <RECORD> id
+ *
+ * @param recordID
+ * @return String[]
+ */
+ String[] getColumns(final String recordID);
+
+ /**
+ * Returns the line number the pointer is on. These are the actual line
+ * numbers from the flat file, before any sorting.
+ *
+ * @exception NoSuchElementException
+ * @exception NumberFormatException
+ * @return int
+ */
+ int getRowNo();
+
+ /**
+ * Returns A Collection Of DataErrors that happened during processing
+ *
+ * @return Vector
+ */
+ List getErrors();
+
+ /**
+ * Removes a row from the dataset. Once the row is removed the pointer will
+ * be sitting on the record previous to the deleted row.
+ */
+ void remove();
+
+ /**
+ * Returns the index the pointer is on for the array
+ *
+ * @return int
+ */
+ int getIndex();
+
+ /**
+ * Returns the total number of rows parsed in from the file
+ *
+ *
+ * @return int - Row Count
+ */
+ int getRowCount();
+
+ /**
+ * Returns total number of records which contained a parse error in the
+ * file.
+ *
+ * @return int - Record Error Count
+ */
+ int getErrorCount();
+
+ /**
+ * Returns true or false as to whether or not the line number contains an
+ * error. The import will skip the line if it contains an error and it will
+ * not be processed
+ *
+ * @param lineNo -
+ * int line number
+ * @return boolean
+ */
+ boolean isAnError(final int lineNo);
+
+ /**
+ * Orders the data by column(s) specified. This will reposition the cursor
+ * to the top of the DataSet when executed. This is currently not supported
+ * when specying <RECORD> elements in the mapping. An exception will be
+ * thrown if this situation occurs
+ *
+ * @param ob -
+ * OrderBy object
+ * @exception Exception
+ * @see net.sf.pzfilereader.ordering.OrderBy
+ * @see net.sf.pzfilereader.ordering.OrderColumn
+ */
+ void orderRows(final OrderBy ob) throws Exception;
+
+ List getRows();
+
+}
\ No newline at end of file
Added: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/PZParser.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/PZParser.java (rev 0)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/PZParser.java 2006-10-26 14:04:40 UTC (rev 125)
@@ -0,0 +1,64 @@
+/*
+ * ObjectLab, http://www.objectlab.co.uk/open is supporting PZFileReader.
+ *
+ * Based in London, we are world leaders in the design and development
+ * of bespoke applications for the securities financing markets.
+ *
+ * <a href="http://www.objectlab.co.uk/open">Click here to learn more</a>
+ * ___ _ _ _ _ _
+ * / _ \| |__ (_) ___ ___| |_| | __ _| |__
+ * | | | | '_ \| |/ _ \/ __| __| | / _` | '_ \
+ * | |_| | |_) | | __/ (__| |_| |__| (_| | |_) |
+ * \___/|_.__// |\___|\___|\__|_____\__,_|_.__/
+ * |__/
+ *
+ * www.ObjectLab.co.uk
+ *
+ * $Id: ColorProvider.java 74 2006-10-24 22:19:05Z benoitx $
+ *
+ * Copyright 2006 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package net.sf.pzfilereader;
+
+/**
+ * PZParser is ready to parse the data and return an object that can then be traversed.
+ * The default parser should NOT handle short lines, the user can change it prior to calling
+ * parse.
+ *
+ * @author Benoit Xhenseval
+ */
+public interface PZParser {
+
+ /**
+ * Start the parsing
+ * @return the data set resulting from parsing
+ */
+ IDataSet parse();
+
+ /**
+ * @return true, lines with less columns then the amount
+ * of column headers will be added as empty's instead of
+ * producing an error
+ */
+ boolean isHandlingShortLines();
+
+ /**
+ * @param handleShortLines -
+ * when flaged as true, lines with less columns then the amount
+ * of column headers will be added as empty's instead of
+ * producing an error
+ */
+ void setHandlingShortLines(final boolean handleShortLines);
+}
Added: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/PZParserFactory.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/PZParserFactory.java (rev 0)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/PZParserFactory.java 2006-10-26 14:04:40 UTC (rev 125)
@@ -0,0 +1,231 @@
+/*
+ * ObjectLab, http://www.objectlab.co.uk/open is supporting PZFileReader.
+ *
+ * Based in London, we are world leaders in the design and development
+ * of bespoke applications for the securities financing markets.
+ *
+ * <a href="http://www.objectlab.co.uk/open">Click here to learn more</a>
+ * ___ _ _ _ _ _
+ * / _ \| |__ (_) ___ ___| |_| | __ _| |__
+ * | | | | '_ \| |/ _ \/ __| __| | / _` | '_ \
+ * | |_| | |_) | | __/ (__| |_| |__| (_| | |_) |
+ * \___/|_.__// |\___|\___|\__|_____\__,_|_.__/
+ * |__/
+ *
+ * www.ObjectLab.co.uk
+ *
+ * $Id: ColorProvider.java 74 2006-10-24 22:19:05Z benoitx $
+ *
+ * Copyright 2006 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package net.sf.pzfilereader;
+
+import java.io.File;
+import java.io.InputStream;
+import java.sql.Connection;
+
+/**
+ * Factory definitions for creating a PZParser (delimiter or fixed length). The
+ * creation of a parser will not start the parsing. It should not fail either
+ * (unless DB issues etc).
+ *
+ * @author Benoit Xhenseval
+ */
+public interface PZParserFactory {
+ /**
+ * Constructs a new DataSet using the database table file layout method.
+ * This is used for a FIXED LENGTH text file.
+ *
+ * The user is responsible for closing the DB connection.
+ *
+ * @param con -
+ * Connection to database with DATAFILE and DATASTRUCTURE tables,
+ * user is responsible for closing it.
+ * @param dataSource -
+ * Fixed length file to read from
+ * @param dataDefinition -
+ * Name of dataDefinition in the DATAFILE table DATAFILE_DESC
+ * column
+ */
+ PZParser newParser(final Connection con, final File dataSource, final String dataDefinition);
+
+ /**
+ * Constructs a new DataSet using the database table file layout method.
+ * This is used for a FIXED LENGTH text file.
+ *
+ * The user is responsible for closing the DB connection and InputStream.
+ *
+ * @param con -
+ * Connection to database with DATAFILE and DATASTRUCTURE tables,
+ * user is responsible for closing it.
+ * @param dataSourceStream -
+ * text file datasource InputStream to read from
+ * @param dataDefinition -
+ * Name of dataDefinition in the DATAFILE table DATAFILE_DESC
+ * column
+ */
+ PZParser newParser(final Connection con, final InputStream dataSourceStream, final String dataDefinition);
+
+ /**
+ * Constructs a new DataSet using the PZMAP XML file layout method. This is
+ * used for a FIXED LENGTH text file.
+ *
+ * @param pzmapXML -
+ * Reference to the xml file holding the pzmap
+ * @param dataSource -
+ * Delimited file to read from
+ */
+ PZParser newParser(final File pzmapXML, final File dataSource);
+
+ /**
+ * New constructor based on InputStream. Constructs a new DataSet using the
+ * PZMAP XML file layout method. This is used for a FIXED LENGTH text file.
+ *
+ * The user is responsible for closing the InputStreams.
+ *
+ * @param pzmapXMLStream -
+ * Reference to the xml file InputStream holding the pzmap, user
+ * must close them after use.
+ * @param dataSourceStream -
+ * Delimited file InputStream to read from, user must close them
+ * after use.
+ */
+ PZParser newParser(final InputStream pzmapXMLStream, final InputStream dataSourceStream);
+
+ //
+ //
+ // ------------------------------------------ DELIMITED -----------
+ //
+ //
+
+ /**
+ * New constructor based on InputStream. Constructs a new DataSet using the
+ * database table file layout method. This is used for a DELIMITED text
+ * file. esacpe sequence reference: \n newline <br>
+ * \t tab <br>
+ * \b backspace <br>
+ * \r return <br>
+ * \f form feed <br> \\ backslash <br> \' single quote <br> \" double quote
+ *
+ * The user is responsible for closing the DB connection and InputStream.
+ *
+ * @param con -
+ * Connection to database with DATAFILE and DATASTRUCTURE tables,
+ * user must close it when done.
+ * @param dataSourceStream -
+ * text file datasource InputStream to read from, user must close
+ * it when done.
+ * @param dataDefinition -
+ * Name of dataDefinition in the DATAFILE table DATAFILE_DESC
+ * column
+ * @param delimiter -
+ * Char the file is delimited By
+ * @param qualifier -
+ * Char text is qualified by
+ * @param ignoreFirstRecord -
+ * skips the first line that contains data in the file
+ */
+ PZParser newParser(final Connection con, final InputStream dataSourceStream, final String dataDefinition,
+ final char delimiter, final char qualifier, final boolean ignoreFirstRecord);
+
+ /**
+ * Constructs a new DataSet using the PZMAP XML file layout method. This is
+ * used for a DELIMITED text file. esacpe sequence reference: \n newline
+ * <br>
+ * \t tab <br>
+ * \b backspace <br>
+ * \r return <br>
+ * \f form feed <br> \\ backslash <br> \' single quote <br> \" double quote
+ *
+ * @param pzmapXML -
+ * Reference to the xml file holding the pzmap
+ * @param dataSource -
+ * text file datasource to read from
+ * @param delimiter -
+ * Char the file is delimited By
+ * @param qualifier -
+ * Char text is qualified by
+ * @param ignoreFirstRecord -
+ * skips the first line that contains data in the file
+ */
+ PZParser newParser(final File pzmapXML, final File dataSource, final char delimiter, final char qualifier,
+ final boolean ignoreFirstRecord);
+
+ /**
+ * New constructor based on InputStream. Constructs a new DataSet using the
+ * PZMAP XML file layout method. This is used for a DELIMITED text file.
+ * esacpe sequence reference: \n newline <br>
+ * \t tab <br>
+ * \b backspace <br>
+ * \r return <br>
+ * \f form feed <br> \\ backslash <br> \' single quote <br> \" double quote
+ *
+ * The user is responsible for closing the InputStreams.
+ *
+ * @param pzmapXMLStream -
+ * Reference to the xml file holding the pzmap, user must close
+ * it when done.
+ * @param dataSourceStream -
+ * text file datasource InputStream to read from, user must close
+ * it when done.
+ * @param delimiter -
+ * Char the file is delimited By
+ * @param qualifier -
+ * Char text is qualified by
+ * @param ignoreFirstRecord -
+ * skips the first line that contains data in the file
+ */
+ PZParser newParser(final InputStream pzmapXMLStream, final InputStream dataSourceStream, final char delimiter,
+ final char qualifier, final boolean ignoreFirstRecord);
+
+ /**
+ * Constructs a new DataSet using the first line of data found in the text
+ * file as the column names. This is used for a DELIMITED text file. esacpe
+ * sequence reference: \n newline <br>
+ * \t tab <br>
+ * \b backspace <br>
+ * \r return <br>
+ * \f form feed <br> \\ backslash <br> \' single quote <br> \" double quote
+ *
+ * @param dataSource -
+ * text file datasource to read from
+ * @param delimiter -
+ * Char the file is delimited By
+ * @param qualifier -
+ * Char text is qualified by
+ */
+ PZParser newParser(final File dataSource, final char delimiter, final char qualifier);
+
+ /**
+ * Constructs a new DataSet using the first line of data found in the text
+ * file as the column names. This is used for a DELIMITED text file. esacpe
+ * sequence reference: \n newline <br>
+ * \t tab <br>
+ * \b backspace <br>
+ * \r return <br>
+ * \f form feed <br> \\ backslash <br> \' single quote <br> \" double quote
+ *
+ * The user must close the InputStream when done (after parsing).
+ *
+ * @param dataSource -
+ * text file InputStream to read from, user must close it when
+ * done.
+ * @param delimiter -
+ * Char the file is delimited By
+ * @param qualifier -
+ * Char text is qualified by
+ */
+ PZParser newParser(final InputStream dataSource, final char delimiter, final char qualifier);
+}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <be...@us...> - 2006-10-26 08:58:13
|
Revision: 124
http://svn.sourceforge.net/pzfilereader/?rev=124&view=rev
Author: benoitx
Date: 2006-10-26 01:58:08 -0700 (Thu, 26 Oct 2006)
Log Message:
-----------
Removed System.out
Modified Paths:
--------------
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java
Modified: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java 2006-10-25 11:36:46 UTC (rev 123)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java 2006-10-26 08:58:08 UTC (rev 124)
@@ -1124,7 +1124,6 @@
for (int i = 0; i < cmds.size(); i++) {
column = (ColumnMetaData) cmds.get(i);
array[i] = column.getColName();
- System.out.println(i + "/ Columns... " + column.getColName());
}
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <zep...@us...> - 2006-10-25 11:37:10
|
Revision: 123
http://svn.sourceforge.net/pzfilereader/?rev=123&view=rev
Author: zepernick
Date: 2006-10-25 04:36:46 -0700 (Wed, 25 Oct 2006)
Log Message:
-----------
removed freeMemory() call and updated constructor
I have not tested the changes yet. I go back through and make sure they are still okay
Modified Paths:
--------------
trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/delimitedcolumnnamesinfile/DelimitedColumnNamesInFile.java
trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/delimiteddynamiccolumns/DelimitedWithPZMap.java
trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/delimiteddynamiccolumnswitherrors/DelimitedWithPZMapErrors.java
trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/exporttoexcel/DelimitedFileExportToExcel.java
trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/fixedlengthdynamiccolumns/FixedLengthWithPZMap.java
trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/fixedlengthheaderandtrailer/FixedLengthHeaderAndTrailer.java
trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/jsptableexample/pzfilereaderwebsamples/index.jsp
trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/largedataset/delimiteddynamiccolumns/LargeDelimitedWithPZMap.java
trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/largedataset/largecsvperformancetest/CSVLarge.java
trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/lowlevelparse/LowLevelParse.java
trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/multilinedelimitedrecord/DelimitedMultiLine.java
trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/numericsanddates/NumericsAndDates.java
Modified: trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/delimitedcolumnnamesinfile/DelimitedColumnNamesInFile.java
===================================================================
--- trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/delimitedcolumnnamesinfile/DelimitedColumnNamesInFile.java 2006-10-25 11:28:47 UTC (rev 122)
+++ trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/delimitedcolumnnamesinfile/DelimitedColumnNamesInFile.java 2006-10-25 11:36:46 UTC (rev 123)
@@ -27,7 +27,7 @@
// delimited by a comma
// text qualified by double quotes
// ignore first record
- ds = new DataSet(new File("PEOPLE-CommaDelimitedWithQualifier.txt"), ",", "\"", false);
+ ds = new DataSet(new File("PEOPLE-CommaDelimitedWithQualifier.txt"), ',', '"', false);
// re order the data set by last name
orderby = new OrderBy();
@@ -53,8 +53,5 @@
}
}
- // clear out the DataSet object for the JVM to collect
- ds.freeMemory();
-
}
}
Modified: trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/delimiteddynamiccolumns/DelimitedWithPZMap.java
===================================================================
--- trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/delimiteddynamiccolumns/DelimitedWithPZMap.java 2006-10-25 11:28:47 UTC (rev 122)
+++ trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/delimiteddynamiccolumns/DelimitedWithPZMap.java 2006-10-25 11:36:46 UTC (rev 123)
@@ -40,7 +40,7 @@
// ignore first record
DataSet ds = null;
OrderBy orderby = null;
- ds = new DataSet(new File(mapping), new File(data), ",", "\"", true, false);
+ ds = new DataSet(new File(mapping), new File(data), ',', '"', true, false);
// re order the data set by last name
orderby = new OrderBy();
@@ -79,7 +79,5 @@
System.out.println("FOUND ERRORS IN FILE");
}
- // clear out the DataSet object for the JVM to collect
- ds.freeMemory();
}
}
Modified: trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/delimiteddynamiccolumnswitherrors/DelimitedWithPZMapErrors.java
===================================================================
--- trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/delimiteddynamiccolumnswitherrors/DelimitedWithPZMapErrors.java 2006-10-25 11:28:47 UTC (rev 122)
+++ trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/delimiteddynamiccolumnswitherrors/DelimitedWithPZMapErrors.java 2006-10-25 11:36:46 UTC (rev 123)
@@ -46,7 +46,7 @@
// delimited by a comma
// text qualified by double quotes
// ignore first record
- ds = new DataSet(new File(mapping), new File(data), ",", "\"", true, false);
+ ds = new DataSet(new File(mapping), new File(data), ',', '"', true, false);
// re order the data set by last name
orderby = new OrderBy();
@@ -91,8 +91,5 @@
System.out.println("ERROR: " + dataError.getErrorDesc() + " LINE NUMBER: " + dataError.getLineNo());
}
- // clear out the DataSet object for the JVM to collect
- ds.freeMemory();
-
}
}
Modified: trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/exporttoexcel/DelimitedFileExportToExcel.java
===================================================================
--- trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/exporttoexcel/DelimitedFileExportToExcel.java 2006-10-25 11:28:47 UTC (rev 122)
+++ trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/exporttoexcel/DelimitedFileExportToExcel.java 2006-10-25 11:36:46 UTC (rev 123)
@@ -41,7 +41,7 @@
// delimited by a comma
// text qualified by double quotes
// ignore first record
- ds = new DataSet(new File(mapping), new File(data), ",", "\"", true, false);
+ ds = new DataSet(new File(mapping), new File(data), ',', '"', true, false);
// re order the data set by last name
orderby = new OrderBy();
@@ -61,8 +61,5 @@
ds.writeToExcel(xlFile);
System.out.println("Excel Workbook Written To: " + xlFile.getAbsolutePath());
- // clear out the DataSet object for the JVM to collect
- ds.freeMemory();
-
}
}
Modified: trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/fixedlengthdynamiccolumns/FixedLengthWithPZMap.java
===================================================================
--- trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/fixedlengthdynamiccolumns/FixedLengthWithPZMap.java 2006-10-25 11:28:47 UTC (rev 122)
+++ trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/fixedlengthdynamiccolumns/FixedLengthWithPZMap.java 2006-10-25 11:36:46 UTC (rev 123)
@@ -47,8 +47,5 @@
System.out.println("===========================================================================");
}
- // clear out the DataSet object for the JVM to collect
- ds.freeMemory();
-
}
}
Modified: trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/fixedlengthheaderandtrailer/FixedLengthHeaderAndTrailer.java
===================================================================
--- trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/fixedlengthheaderandtrailer/FixedLengthHeaderAndTrailer.java 2006-10-25 11:28:47 UTC (rev 122)
+++ trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/fixedlengthheaderandtrailer/FixedLengthHeaderAndTrailer.java 2006-10-25 11:36:46 UTC (rev 123)
@@ -78,8 +78,5 @@
}
}
- // clear out the DataSet object for the JVM to collect
- ds.freeMemory();
-
}
}
Modified: trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/jsptableexample/pzfilereaderwebsamples/index.jsp
===================================================================
--- trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/jsptableexample/pzfilereaderwebsamples/index.jsp 2006-10-25 11:28:47 UTC (rev 122)
+++ trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/jsptableexample/pzfilereaderwebsamples/index.jsp 2006-10-25 11:36:46 UTC (rev 123)
@@ -26,8 +26,8 @@
//read in the file a
ds = new DataSet(mappingFile, //mapping file here
txtFile, //text file that is being parsed
- ",", //how the file is delimited
- "", //if the text is qualified, pass the qualifier here, otherwise leave empty
+ ',', //how the file is delimited
+ 0, //if the text is qualified, pass the qualifier here, otherwise leave empty
false, //ignore the first row in the file if it contain column names
false); //don't auto add missing columns, log them as errors
@@ -110,9 +110,6 @@
<%
}
-
- //clear out the DataSet object for the JVM to collect
- ds.freeMemory();
%>
</table>
Modified: trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/largedataset/delimiteddynamiccolumns/LargeDelimitedWithPZMap.java
===================================================================
--- trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/largedataset/delimiteddynamiccolumns/LargeDelimitedWithPZMap.java 2006-10-25 11:28:47 UTC (rev 122)
+++ trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/largedataset/delimiteddynamiccolumns/LargeDelimitedWithPZMap.java 2006-10-25 11:36:46 UTC (rev 123)
@@ -43,7 +43,7 @@
// delimited by a comma
// text qualified by double quotes
// ignore first record
- ds = new LargeDataSet(pzmap, fileToParse, ",", "\"", true, false);
+ ds = new LargeDataSet(pzmap, fileToParse, ',', '"', true, false);
colNames = ds.getColumns();
Modified: trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/largedataset/largecsvperformancetest/CSVLarge.java
===================================================================
--- trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/largedataset/largecsvperformancetest/CSVLarge.java 2006-10-25 11:28:47 UTC (rev 122)
+++ trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/largedataset/largecsvperformancetest/CSVLarge.java 2006-10-25 11:36:46 UTC (rev 123)
@@ -47,7 +47,7 @@
// delimited by a comma
// text qualified by double quotes
// ignore first record
- ds = new LargeDataSet(new File(data), ",", "\"", false);
+ ds = new LargeDataSet(new File(data), ',', '"', false);
final long timeStarted = System.currentTimeMillis();
int totalCount = 0;
Modified: trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/lowlevelparse/LowLevelParse.java
===================================================================
--- trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/lowlevelparse/LowLevelParse.java 2006-10-25 11:28:47 UTC (rev 122)
+++ trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/lowlevelparse/LowLevelParse.java 2006-10-25 11:36:46 UTC (rev 123)
@@ -52,7 +52,7 @@
// text qualifier. The text qualifier is optional, it can be
// null
// or empty
- elements = ParserUtils.splitLine(line, ",", "\"");
+ elements = ParserUtils.splitLine(line, ',', '"');
for (int i = 0; i < elements.size(); i++) {
System.out.println("Column " + i + ": " + (String) elements.get(i));
Modified: trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/multilinedelimitedrecord/DelimitedMultiLine.java
===================================================================
--- trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/multilinedelimitedrecord/DelimitedMultiLine.java 2006-10-25 11:28:47 UTC (rev 122)
+++ trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/multilinedelimitedrecord/DelimitedMultiLine.java 2006-10-25 11:36:46 UTC (rev 123)
@@ -37,7 +37,7 @@
// delimited by a comma
// text qualified by double quotes
// ignore first record
- ds = new DataSet(new File(data), ",", "\"", false);
+ ds = new DataSet(new File(data), ',', '"', false);
colNames = ds.getColumns();
@@ -53,8 +53,5 @@
System.out.println("FOUND ERRORS IN FILE");
}
- // clear out the DataSet object for the JVM to collect
- ds.freeMemory();
-
}
}
Modified: trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/numericsanddates/NumericsAndDates.java
===================================================================
--- trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/numericsanddates/NumericsAndDates.java 2006-10-25 11:28:47 UTC (rev 122)
+++ trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/numericsanddates/NumericsAndDates.java 2006-10-25 11:36:46 UTC (rev 123)
@@ -39,7 +39,7 @@
// delimited by a comma
// text qualified by double quotes
// ignore first record
- ds = new DataSet(new File(mapping), new File(data), ",", "\"", true, false);
+ ds = new DataSet(new File(mapping), new File(data), ',', '"', true, false);
// demonstrates the casting abilities of PZFileReader
while (ds.next()) {
@@ -49,9 +49,5 @@
System.out.println("Received Dt: " + sdf.format(ds.getDate("LAST_RECV_DT")) + " (Date)");
System.out.println("===========================================================================");
}
-
- // clear out the DataSet object for the JVM to collect
- ds.freeMemory();
-
}
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <zep...@us...> - 2006-10-25 11:28:53
|
Revision: 122
http://svn.sourceforge.net/pzfilereader/?rev=122&view=rev
Author: zepernick
Date: 2006-10-25 04:28:47 -0700 (Wed, 25 Oct 2006)
Log Message:
-----------
removed freeMemmory call and updated constructor
Modified Paths:
--------------
trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/csvperformancetest/CSVPerformanceTest.java
Modified: trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/csvperformancetest/CSVPerformanceTest.java
===================================================================
--- trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/csvperformancetest/CSVPerformanceTest.java 2006-10-25 11:27:49 UTC (rev 121)
+++ trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/csvperformancetest/CSVPerformanceTest.java 2006-10-25 11:28:47 UTC (rev 122)
@@ -49,7 +49,7 @@
// ignore first record
System.out.println("Parsing....");
long timeStarted = System.currentTimeMillis();
- ds = new DataSet(new File(filename), ",", "\"", false);
+ ds = new DataSet(new File(filename), ',', '"', false);
long timeFinished = System.currentTimeMillis();
String timeMessage = "";
@@ -108,8 +108,6 @@
}
}
- // clear out the DataSet object for the JVM to collect
- ds.freeMemory();
}
private static Map readSettings() throws Exception {
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <zep...@us...> - 2006-10-25 11:27:52
|
Revision: 121
http://svn.sourceforge.net/pzfilereader/?rev=121&view=rev
Author: zepernick
Date: 2006-10-25 04:27:49 -0700 (Wed, 25 Oct 2006)
Log Message:
-----------
removed freeMemmory call and updated constructor
Modified Paths:
--------------
trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/csvheaderandtrailer/CSVHeaderAndTrailer.java
Modified: trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/csvheaderandtrailer/CSVHeaderAndTrailer.java
===================================================================
--- trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/csvheaderandtrailer/CSVHeaderAndTrailer.java 2006-10-25 11:26:56 UTC (rev 120)
+++ trunk/PZFileReaderSamples/src/main/java/net/sf/pzfilereader/examples/csvheaderandtrailer/CSVHeaderAndTrailer.java 2006-10-25 11:27:49 UTC (rev 121)
@@ -34,7 +34,7 @@
// delimited by a comma
// text qualified by double quotes
// ignore first record
- DataSet ds = new DataSet(mapFile, dataFile, ",", "\"", true, false);
+ DataSet ds = new DataSet(mapFile, dataFile, ',', '"', true, false);
while (ds.next()) {
@@ -66,8 +66,5 @@
if (ds.getErrors() != null && ds.getErrors().size() > 0) {
System.out.println("FOUND ERRORS IN FILE");
}
-
- // clear out the DataSet object for the JVM to collect
- ds.freeMemory();
}
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <zep...@us...> - 2006-10-25 11:27:00
|
Revision: 120
http://svn.sourceforge.net/pzfilereader/?rev=120&view=rev
Author: zepernick
Date: 2006-10-25 04:26:56 -0700 (Wed, 25 Oct 2006)
Log Message:
-----------
added missing char version of constructor
Modified Paths:
--------------
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java
Modified: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java 2006-10-24 23:33:29 UTC (rev 119)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java 2006-10-25 11:26:56 UTC (rev 120)
@@ -390,7 +390,38 @@
.charAt(0) : 0, ignoreFirstRecord, handleShortLines);
}
+
/**
+ * Constructs a new DataSet using the PZMAP XML file layout method. This is
+ * used for a DELIMITED text file. esacpe sequence reference: \n newline
+ * <br>
+ * \t tab <br>
+ * \b backspace <br>
+ * \r return <br>
+ * \f form feed <br> \\ backslash <br> \' single quote <br> \" double quote
+ *
+ * @param pzmapXML -
+ * Reference to the xml file holding the pzmap
+ * @param dataSource -
+ * text file datasource to read from
+ * @param delimiter -
+ * Char the file is delimited By
+ * @param qualifier -
+ * Char text is qualified by
+ * @param ignoreFirstRecord -
+ * skips the first line that contains data in the file
+ * @param handleShortLines -
+ * Adds missing columns as empty's to the DataSet instead of
+ * logging them as an error
+ * @exception Exception
+ */
+ public DataSet(final File pzmapXML, final File dataSource, final char delimiter, final char qualifier,
+ final boolean ignoreFirstRecord, final boolean handleShortLines) throws Exception {
+ this(ParserUtils.createInputStream(pzmapXML), ParserUtils.createInputStream(dataSource), delimiter, qualifier,
+ ignoreFirstRecord, handleShortLines);
+ }
+
+ /**
* New constructor based on InputStream. Constructs a new DataSet using the
* PZMAP XML file layout method. This is used for a DELIMITED text file.
* esacpe sequence reference: \n newline <br>
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <be...@us...> - 2006-10-24 23:33:52
|
Revision: 119
http://svn.sourceforge.net/pzfilereader/?rev=119&view=rev
Author: benoitx
Date: 2006-10-24 16:33:29 -0700 (Tue, 24 Oct 2006)
Log Message:
-----------
Optimised the ParserUtils to use char for delimiter and qualifier.
I have added deprecated methods for Strings (using only the first character). Could you find out where these are used and remove the call to those and use the char instead.
All tests are passed but we should add more... especially with regards to the multi line one...
Time to hit the sack!
Modified Paths:
--------------
trunk/PZFileReader/qalab.xml
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/LargeDataSet.java
trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java
trunk/PZFileReader/src/test/java/net/sf/pzfilereader/parserutils/ParserUtilsSplitLineTest.java
trunk/PZFileReader/src/test/java/net/sf/pzfilereader/utilities/UnitTestUtils.java
trunk/PZFileReaderSamples/qalab.xml
Property Changed:
----------------
trunk/
trunk/PZFileReaderSamples/
Property changes on: trunk
___________________________________________________________________
Name: svn:ignore
- target
+ target
*.log
Modified: trunk/PZFileReader/qalab.xml
===================================================================
--- trunk/PZFileReader/qalab.xml 2006-10-24 21:36:11 UTC (rev 118)
+++ trunk/PZFileReader/qalab.xml 2006-10-24 23:33:29 UTC (rev 119)
@@ -59,6 +59,18 @@
<summaryresult date="2006-10-18" filecount="3" statvalue="116" type="simian"/>
<summaryresult date="2006-10-18" filecount="14" statvalue="18" type="cobertura-line"/>
<summaryresult date="2006-10-18" filecount="14" statvalue="24" type="cobertura-branch"/>
+ <summaryresult date="2006-10-20" filecount="19" statvalue="26" type="checkstyle"/>
+ <summaryresult date="2006-10-20" filecount="3" statvalue="5" type="pmd"/>
+ <summaryresult date="2006-10-20" filecount="6" statvalue="11" type="findbugs"/>
+ <summaryresult date="2006-10-20" filecount="3" statvalue="116" type="simian"/>
+ <summaryresult date="2006-10-20" filecount="14" statvalue="18" type="cobertura-line"/>
+ <summaryresult date="2006-10-20" filecount="14" statvalue="24" type="cobertura-branch"/>
+ <summaryresult date="2006-10-25" filecount="19" statvalue="26" type="checkstyle"/>
+ <summaryresult date="2006-10-25" filecount="3" statvalue="6" type="pmd"/>
+ <summaryresult date="2006-10-25" filecount="6" statvalue="12" type="findbugs"/>
+ <summaryresult date="2006-10-25" filecount="3" statvalue="106" type="simian"/>
+ <summaryresult date="2006-10-25" filecount="14" statvalue="20" type="cobertura-line"/>
+ <summaryresult date="2006-10-25" filecount="14" statvalue="27" type="cobertura-branch"/>
</summary>
<file id="com_pz_reader_ordering_package.html" path="com/pz/reader/ordering/package.html">
<result date="2006-10-05" statvalue="1" type="checkstyle"/>
@@ -337,6 +349,10 @@
<result date="2006-10-17" statvalue="100" type="cobertura-branch"/>
<result date="2006-10-18" statvalue="3" type="checkstyle"/>
<result date="2006-10-18" statvalue="100" type="cobertura-branch"/>
+ <result date="2006-10-20" statvalue="3" type="checkstyle"/>
+ <result date="2006-10-20" statvalue="100" type="cobertura-branch"/>
+ <result date="2006-10-25" statvalue="3" type="checkstyle"/>
+ <result date="2006-10-25" statvalue="100" type="cobertura-branch"/>
</file>
<file id="net_sf_pzfilereader_DataSet.java" path="net/sf/pzfilereader/DataSet.java">
<result date="2006-10-17" statvalue="4" type="checkstyle"/>
@@ -349,16 +365,32 @@
<result date="2006-10-18" statvalue="2" type="findbugs"/>
<result date="2006-10-18" statvalue="21" type="cobertura-line"/>
<result date="2006-10-18" statvalue="27" type="cobertura-branch"/>
+ <result date="2006-10-20" statvalue="5" type="checkstyle"/>
+ <result date="2006-10-20" statvalue="1" type="pmd"/>
+ <result date="2006-10-20" statvalue="2" type="findbugs"/>
+ <result date="2006-10-20" statvalue="21" type="cobertura-line"/>
+ <result date="2006-10-20" statvalue="27" type="cobertura-branch"/>
+ <result date="2006-10-25" statvalue="5" type="checkstyle"/>
+ <result date="2006-10-25" statvalue="1" type="pmd"/>
+ <result date="2006-10-25" statvalue="2" type="findbugs"/>
+ <result date="2006-10-25" statvalue="21" type="cobertura-line"/>
+ <result date="2006-10-25" statvalue="27" type="cobertura-branch"/>
</file>
<file id="net_sf_pzfilereader_LargeDataSet.java" path="net/sf/pzfilereader/LargeDataSet.java">
<result date="2006-10-17" statvalue="4" type="checkstyle"/>
<result date="2006-10-18" statvalue="4" type="checkstyle"/>
+ <result date="2006-10-20" statvalue="4" type="checkstyle"/>
+ <result date="2006-10-25" statvalue="4" type="checkstyle"/>
</file>
<file id="net_sf_pzfilereader_ordering_OrderBy.java" path="net/sf/pzfilereader/ordering/OrderBy.java">
<result date="2006-10-17" statvalue="1" type="checkstyle"/>
<result date="2006-10-17" statvalue="2" type="findbugs"/>
<result date="2006-10-18" statvalue="1" type="checkstyle"/>
<result date="2006-10-18" statvalue="2" type="findbugs"/>
+ <result date="2006-10-20" statvalue="1" type="checkstyle"/>
+ <result date="2006-10-20" statvalue="2" type="findbugs"/>
+ <result date="2006-10-25" statvalue="1" type="checkstyle"/>
+ <result date="2006-10-25" statvalue="2" type="findbugs"/>
</file>
<file id="net_sf_pzfilereader_util_ParserUtils.java" path="net/sf/pzfilereader/util/ParserUtils.java">
<result date="2006-10-17" statvalue="4" type="checkstyle"/>
@@ -371,6 +403,16 @@
<result date="2006-10-18" statvalue="3" type="findbugs"/>
<result date="2006-10-18" statvalue="36" type="cobertura-line"/>
<result date="2006-10-18" statvalue="43" type="cobertura-branch"/>
+ <result date="2006-10-20" statvalue="9" type="checkstyle"/>
+ <result date="2006-10-20" statvalue="3" type="pmd"/>
+ <result date="2006-10-20" statvalue="3" type="findbugs"/>
+ <result date="2006-10-20" statvalue="36" type="cobertura-line"/>
+ <result date="2006-10-20" statvalue="43" type="cobertura-branch"/>
+ <result date="2006-10-25" statvalue="9" type="checkstyle"/>
+ <result date="2006-10-25" statvalue="4" type="pmd"/>
+ <result date="2006-10-25" statvalue="4" type="findbugs"/>
+ <result date="2006-10-25" statvalue="43" type="cobertura-line"/>
+ <result date="2006-10-25" statvalue="52" type="cobertura-branch"/>
</file>
<file id="net_sf_pzfilereader_xml_PZMapParser.java" path="net/sf/pzfilereader/xml/PZMapParser.java">
<result date="2006-10-17" statvalue="4" type="checkstyle"/>
@@ -379,31 +421,51 @@
<result date="2006-10-18" statvalue="4" type="checkstyle"/>
<result date="2006-10-18" statvalue="1" type="pmd"/>
<result date="2006-10-18" statvalue="1" type="findbugs"/>
+ <result date="2006-10-20" statvalue="4" type="checkstyle"/>
+ <result date="2006-10-20" statvalue="1" type="pmd"/>
+ <result date="2006-10-20" statvalue="1" type="findbugs"/>
+ <result date="2006-10-25" statvalue="4" type="checkstyle"/>
+ <result date="2006-10-25" statvalue="1" type="pmd"/>
+ <result date="2006-10-25" statvalue="1" type="findbugs"/>
</file>
<file id="net_sf_pzfilereader_xml_ResolveLocalDTD.java" path="net/sf/pzfilereader/xml/ResolveLocalDTD.java">
<result date="2006-10-17" statvalue="2" type="findbugs"/>
<result date="2006-10-18" statvalue="2" type="findbugs"/>
+ <result date="2006-10-20" statvalue="2" type="findbugs"/>
+ <result date="2006-10-25" statvalue="2" type="findbugs"/>
</file>
<file id="net_sf_pzfilereader_structure_Row.java" path="net/sf/pzfilereader/structure/Row.java">
<result date="2006-10-17" statvalue="1" type="findbugs"/>
<result date="2006-10-17" statvalue="44" type="cobertura-line"/>
<result date="2006-10-18" statvalue="1" type="findbugs"/>
<result date="2006-10-18" statvalue="44" type="cobertura-line"/>
+ <result date="2006-10-20" statvalue="1" type="findbugs"/>
+ <result date="2006-10-20" statvalue="44" type="cobertura-line"/>
+ <result date="2006-10-25" statvalue="1" type="findbugs"/>
+ <result date="2006-10-25" statvalue="44" type="cobertura-line"/>
</file>
<file id="net_sf_pzfilereader_structure_ColumnMetaData.java" path="net/sf/pzfilereader/structure/ColumnMetaData.java">
<result date="2006-10-17" statvalue="50" type="cobertura-line"/>
<result date="2006-10-17" statvalue="100" type="cobertura-branch"/>
<result date="2006-10-18" statvalue="50" type="cobertura-line"/>
<result date="2006-10-18" statvalue="100" type="cobertura-branch"/>
+ <result date="2006-10-20" statvalue="50" type="cobertura-line"/>
+ <result date="2006-10-20" statvalue="100" type="cobertura-branch"/>
+ <result date="2006-10-25" statvalue="50" type="cobertura-line"/>
+ <result date="2006-10-25" statvalue="100" type="cobertura-branch"/>
</file>
<file id="net_sf_pzfilereader_Version.java" path="net/sf/pzfilereader/Version.java">
<result date="2006-10-17" statvalue="100" type="cobertura-branch"/>
<result date="2006-10-18" statvalue="100" type="cobertura-branch"/>
+ <result date="2006-10-20" statvalue="100" type="cobertura-branch"/>
+ <result date="2006-10-25" statvalue="100" type="cobertura-branch"/>
</file>
<file id="net_sf_pzfilereader_xml_XMLRecordElement.java" path="net/sf/pzfilereader/xml/XMLRecordElement.java">
<result date="2006-10-17" statvalue="100" type="cobertura-branch"/>
</file>
<file id="net_sf_pzfilereader_util_PZConstants.java" path="net/sf/pzfilereader/util/PZConstants.java">
<result date="2006-10-18" statvalue="100" type="cobertura-branch"/>
+ <result date="2006-10-20" statvalue="100" type="cobertura-branch"/>
+ <result date="2006-10-25" statvalue="100" type="cobertura-branch"/>
</file>
</qalab>
Modified: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java 2006-10-24 21:36:11 UTC (rev 118)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/DataSet.java 2006-10-24 23:33:29 UTC (rev 119)
@@ -205,6 +205,7 @@
* Adds missing columns as empty's to the DataSet instead of
* logging them as an error
* @exception Exception
+ * @deprecated use the char version
*/
public DataSet(final Connection con, final File dataSource, final String dataDefinition, final String delimiter,
final String qualifier, final boolean ignoreFirstRecord, final boolean handleShortLines) throws Exception {
@@ -238,9 +239,43 @@
* Adds missing columns as empty's to the DataSet instead of
* logging them as an error
* @exception Exception
+ * @deprecated qualifier and delimiters should only be char.
*/
public DataSet(final Connection con, final InputStream dataSourceStream, final String dataDefinition, final String delimiter,
final String qualifier, final boolean ignoreFirstRecord, final boolean handleShortLines) throws Exception {
+ this(con, dataSourceStream, dataDefinition, delimiter != null ? delimiter.charAt(0) : 0, qualifier != null ? qualifier
+ .charAt(0) : 0, ignoreFirstRecord, handleShortLines);
+ }
+
+ /**
+ * New constructor based on InputStream. Constructs a new DataSet using the
+ * database table file layout method. This is used for a DELIMITED text
+ * file. esacpe sequence reference: \n newline <br>
+ * \t tab <br>
+ * \b backspace <br>
+ * \r return <br>
+ * \f form feed <br> \\ backslash <br> \' single quote <br> \" double quote
+ *
+ * @param con -
+ * Connection to database with DATAFILE and DATASTRUCTURE tables
+ * @param dataSourceStream -
+ * text file datasource InputStream to read from
+ * @param dataDefinition -
+ * Name of dataDefinition in the DATAFILE table DATAFILE_DESC
+ * column
+ * @param delimiter -
+ * Char the file is delimited By
+ * @param qualifier -
+ * Char text is qualified by
+ * @param ignoreFirstRecord -
+ * skips the first line that contains data in the file
+ * @param handleShortLines -
+ * Adds missing columns as empty's to the DataSet instead of
+ * logging them as an error
+ * @exception Exception
+ */
+ public DataSet(final Connection con, final InputStream dataSourceStream, final String dataDefinition, final char delimiter,
+ final char qualifier, final boolean ignoreFirstRecord, final boolean handleShortLines) throws Exception {
super();
this.handleShortLines = handleShortLines;
@@ -316,6 +351,7 @@
* Adds missing columns as empty's to the DataSet instead of
* logging them as an error
* @exception Exception
+ * @deprecated use the char version
*/
public DataSet(final File pzmapXML, final File dataSource, final String delimiter, final String qualifier,
final boolean ignoreFirstRecord, final boolean handleShortLines) throws Exception {
@@ -346,10 +382,41 @@
* Adds missing columns as empty's to the DataSet instead of
* logging them as an error
* @exception Exception
+ * @deprecated use the char version
*/
public DataSet(final InputStream pzmapXMLStream, final InputStream dataSourceStream, final String delimiter,
final String qualifier, final boolean ignoreFirstRecord, final boolean handleShortLines) throws Exception {
+ this(pzmapXMLStream, dataSourceStream, delimiter != null ? delimiter.charAt(0) : 0, qualifier != null ? qualifier
+ .charAt(0) : 0, ignoreFirstRecord, handleShortLines);
+ }
+ /**
+ * New constructor based on InputStream. Constructs a new DataSet using the
+ * PZMAP XML file layout method. This is used for a DELIMITED text file.
+ * esacpe sequence reference: \n newline <br>
+ * \t tab <br>
+ * \b backspace <br>
+ * \r return <br>
+ * \f form feed <br> \\ backslash <br> \' single quote <br> \" double quote
+ *
+ * @param pzmapXMLStream -
+ * Reference to the xml file holding the pzmap
+ * @param dataSourceStream -
+ * text file datasource InputStream to read from
+ * @param delimiter -
+ * Char the file is delimited By
+ * @param qualifier -
+ * Char text is qualified by
+ * @param ignoreFirstRecord -
+ * skips the first line that contains data in the file
+ * @param handleShortLines -
+ * Adds missing columns as empty's to the DataSet instead of
+ * logging them as an error
+ * @exception Exception
+ */
+ public DataSet(final InputStream pzmapXMLStream, final InputStream dataSourceStream, final char delimiter,
+ final char qualifier, final boolean ignoreFirstRecord, final boolean handleShortLines) throws Exception {
+
this.handleShortLines = handleShortLines;
columnMD = PZMapParser.parse(pzmapXMLStream);
@@ -376,10 +443,38 @@
* of column headers will be added as empty's instead of
* producing an error
* @exception Exception
+ * @deprecated
*/
public DataSet(final File dataSource, final String delimiter, final String qualifier, final boolean handleShortLines)
throws Exception {
+ this(dataSource, delimiter != null ? delimiter.charAt(0) : 0, qualifier != null ? qualifier.charAt(0) : 0,
+ handleShortLines);
+ }
+ /**
+ * Constructs a new DataSet using the first line of data found in the text
+ * file as the column names. This is used for a DELIMITED text file. esacpe
+ * sequence reference: \n newline <br>
+ * \t tab <br>
+ * \b backspace <br>
+ * \r return <br>
+ * \f form feed <br> \\ backslash <br> \' single quote <br> \" double quote
+ *
+ * @param dataSource -
+ * text file datasource to read from
+ * @param delimiter -
+ * Char the file is delimited By
+ * @param qualifier -
+ * Char text is qualified by
+ * @param handleShortLines -
+ * when flaged as true, lines with less columns then the amount
+ * of column headers will be added as empty's instead of
+ * producing an error
+ * @exception Exception
+ */
+ public DataSet(final File dataSource, final char delimiter, final char qualifier, final boolean handleShortLines)
+ throws Exception {
+
this.handleShortLines = handleShortLines;
InputStream dataSourceStream = null;
@@ -413,10 +508,38 @@
* of column headers will be added as empty's instead of
* producing an error
* @exception Exception
+ * @deprecated
*/
public DataSet(final InputStream dataSource, final String delimiter, final String qualifier, final boolean handleShortLines)
throws Exception {
+ this(dataSource, delimiter != null ? delimiter.charAt(0) : 0, qualifier != null ? qualifier.charAt(0) : 0,
+ handleShortLines);
+ }
+ /**
+ * Constructs a new DataSet using the first line of data found in the text
+ * file as the column names. This is used for a DELIMITED text file. esacpe
+ * sequence reference: \n newline <br>
+ * \t tab <br>
+ * \b backspace <br>
+ * \r return <br>
+ * \f form feed <br> \\ backslash <br> \' single quote <br> \" double quote
+ *
+ * @param dataSource -
+ * text file InputStream to read from
+ * @param delimiter -
+ * Char the file is delimited By
+ * @param qualifier -
+ * Char text is qualified by
+ * @param handleShortLines -
+ * when flaged as true, lines with less columns then the amount
+ * of column headers will be added as empty's instead of
+ * producing an error
+ * @exception Exception
+ */
+ public DataSet(final InputStream dataSource, final char delimiter, final char qualifier, final boolean handleShortLines)
+ throws Exception {
+
this.handleShortLines = handleShortLines;
try {
@@ -556,7 +679,7 @@
* puts together the dataset for a DELIMITED file. This is used for PZ XML
* mappings, and SQL table mappings
*/
- private void doDelimitedFile(final InputStream dataSource, final String delimiter, final String qualifier,
+ private void doDelimitedFile(final InputStream dataSource, final char delimiter, final char qualifier,
final boolean ignoreFirstRecord, final boolean createMDFromFile) throws Exception {
if (dataSource == null) {
throw new NullPointerException("dataSource is null");
@@ -606,7 +729,7 @@
// be checked if we have specified a delimiter
// ********************************************************
final char[] chrArry = line.trim().toCharArray();
- if (!processingMultiLine && delimiter != null && delimiter.trim().length() > 0) {
+ if (!processingMultiLine && delimiter > 0) {
processingMultiLine = ParserUtils.isMultiLine(chrArry, delimiter, qualifier);
}
@@ -619,7 +742,8 @@
// excel will escape these with another quote; here is some
// data "" This would indicate
// there is more to the multiline
- if (line.trim().endsWith(qualifier) && !line.trim().endsWith(qualifier + qualifier)) {
+ String trimmed = line.trim();
+ if (trimmed.charAt(trimmed.length() - 1) == qualifier && !trimmed.endsWith("" + qualifier + qualifier)) {
// it is safe to assume we have reached the end of the
// line break
processingMultiLine = false;
@@ -645,7 +769,7 @@
// delimiter, then we have reached the end
// of
// the record
- if (chrArry[i] == delimiter.charAt(0)) {
+ if (chrArry[i] == delimiter) {
// processingMultiLine = false;
// fix put in, setting to false caused
// bug when processing multiple
@@ -657,7 +781,7 @@
qualiFound = false;
continue;
}
- } else if (chrArry[i] == qualifier.charAt(0)) {
+ } else if (chrArry[i] == qualifier) {
qualiFound = true;
}
}
@@ -1186,8 +1310,11 @@
}
/**
- * Erases the dataset early and releases memory for the JVM to reclaim, this invalidates the object.
- * @deprecated You can still use it but truly you should keep the scope of the DataSet to a MINIMUM.
+ * Erases the dataset early and releases memory for the JVM to reclaim, this
+ * invalidates the object.
+ *
+ * @deprecated You can still use it but truly you should keep the scope of
+ * the DataSet to a MINIMUM.
*/
public void freeMemory() {
if (rows != null) {
Modified: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/LargeDataSet.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/LargeDataSet.java 2006-10-24 21:36:11 UTC (rev 118)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/LargeDataSet.java 2006-10-24 23:33:29 UTC (rev 119)
@@ -60,9 +60,9 @@
private boolean processedFirst = false;
- private String delimiter = null;
+ private char delimiter = 0;
- private String qualifier = null;
+ private char qualifier = 0;
private int columnCount = 0;
@@ -96,10 +96,42 @@
* Adds missing columns as empty's to the DataSet instead of
* logging them as an error
* @exception Exception
+ * @deprecated use the char version.
*/
public LargeDataSet(final InputStream pzmapXMLStream, final InputStream dataSourceStream, final String delimiter,
final String qualifier, final boolean ignoreFirstRecord, final boolean handleShortLines) throws Exception {
+ this(pzmapXMLStream, dataSourceStream, delimiter != null ? delimiter.charAt(0) : 0, qualifier != null ? qualifier
+ .charAt(0) : 0, ignoreFirstRecord, handleShortLines);
+ }
+ /**
+ * Constructor based on InputStream. Constructs a new LargeDataSet using the
+ * PZMAP XML file layout method. This is used for a DELIMITED text file.
+ * esacpe sequence reference:<br>
+ * \n newline <br>
+ * \t tab <br>
+ * \b backspace <br>
+ * \r return <br>
+ * \f form feed <br> \\ backslash <br> \' single quote <br> \" double quote
+ *
+ * @param pzmapXMLStream -
+ * Reference to the xml file holding the pzmap
+ * @param dataSourceStream -
+ * text file datasource InputStream to read from
+ * @param delimiter -
+ * Char the file is delimited By
+ * @param qualifier -
+ * Char text is qualified by
+ * @param ignoreFirstRecord -
+ * skips the first line that contains data in the file
+ * @param handleShortLines -
+ * Adds missing columns as empty's to the DataSet instead of
+ * logging them as an error
+ * @exception Exception
+ */
+ public LargeDataSet(final InputStream pzmapXMLStream, final InputStream dataSourceStream, final char delimiter,
+ final char qualifier, final boolean ignoreFirstRecord, final boolean handleShortLines) throws Exception {
+
this.fileType = DELIMITED_FILE;
this.is = dataSourceStream;
this.isr = new InputStreamReader(is);
@@ -133,9 +165,38 @@
* of column headers will be added as empty's instead of
* producing an error
* @exception Exception
+ * @deprecated use the char version
*/
public LargeDataSet(final File dataSource, final String delimiter, final String qualifier, final boolean handleShortLines)
throws Exception {
+ this(dataSource, delimiter != null ? delimiter.charAt(0) : 0, qualifier != null ? qualifier.charAt(0) : 0,
+ handleShortLines);
+ }
+
+ /**
+ * Constructs a new LargeDataSet using the first line of data found in the
+ * text file as the column names. This is used for a DELIMITED text file.
+ * esacpe sequence reference:<br>
+ * \n newline <br>
+ * \t tab <br>
+ * \b backspace <br>
+ * \r return <br>
+ * \f form feed <br> \\ backslash <br> \' single quote <br> \" double quote
+ *
+ * @param dataSource -
+ * text file datasource to read from
+ * @param delimiter -
+ * Char the file is delimited By
+ * @param qualifier -
+ * Char text is qualified by
+ * @param handleShortLines -
+ * when flaged as true, lines with less columns then the amount
+ * of column headers will be added as empty's instead of
+ * producing an error
+ * @exception Exception
+ */
+ public LargeDataSet(final File dataSource, final char delimiter, final char qualifier, final boolean handleShortLines)
+ throws Exception {
this(ParserUtils.createInputStream(dataSource), delimiter, qualifier, handleShortLines);
}
@@ -160,9 +221,39 @@
* of column headers will be added as empty's instead of
* producing an error
* @exception Exception
+ * @deprecated use the char version
*/
public LargeDataSet(final InputStream dataSource, final String delimiter, final String qualifier,
final boolean handleShortLines) throws Exception {
+ this(dataSource, delimiter != null ? delimiter.charAt(0) : 0, qualifier != null ? qualifier.charAt(0) : 0,
+ handleShortLines);
+ }
+
+ /**
+ * Constructs a new LargeDataSet using the first line of data found in the
+ * text file as the column names. This is used for a DELIMITED text file.
+ * esacpe sequence reference:<br>
+ * \n newline <br>
+ * \t tab <br>
+ * \b backspace <br>
+ * \r return <br>
+ * \f form feed <br> \\ backslash <br> \' single quote <br> \" double quote
+ *
+ * @param dataSource -
+ * text file datasource to read from
+ * @param delimiter -
+ * Char the file is delimited By
+ * @param qualifier -
+ * Char text is qualified by
+ * @param handleShortLines -
+ * when flaged as true, lines with less columns then the amount
+ * of column headers will be added as empty's instead of
+ * producing an error
+ * @exception Exception
+ */
+ public LargeDataSet(final InputStream dataSource, final char delimiter, final char qualifier, final boolean handleShortLines)
+ throws Exception {
+
this.fileType = DELIMITED_FILE;
setHandleShortLines(handleShortLines);
this.is = dataSource;
@@ -342,8 +433,9 @@
/** loop through each line in the file */
while ((line = br.readLine()) != null) {
lineCount++;
+ String trimmed = line.trim();
/** empty line skip past it */
- if (!processingMultiLine && line.trim().length() == 0) {
+ if (!processingMultiLine && trimmed.length() == 0) {
continue;
}
// check to see if the user has elected to skip the first record
@@ -362,8 +454,8 @@
// any line breaks in the middle of the record, this will only
// be checked if we have specified a delimiter
// ********************************************************
- final char[] chrArry = line.trim().toCharArray();
- if (!processingMultiLine && delimiter != null && delimiter.trim().length() > 0) {
+ final char[] chrArry = trimmed.toCharArray();
+ if (!processingMultiLine && delimiter > 0) {
processingMultiLine = ParserUtils.isMultiLine(chrArry, delimiter, qualifier);
}
@@ -376,7 +468,7 @@
// excel will escape these with another quote; here is some data
// "" This would indicate
// there is more to the multiline
- if (line.trim().endsWith(qualifier) && !line.trim().endsWith(qualifier + qualifier)) {
+ if (trimmed.charAt(trimmed.length() - 1) == qualifier && !trimmed.endsWith("" + qualifier + qualifier)) {
// it is safe to assume we have reached the end of the line
// break
processingMultiLine = false;
@@ -401,7 +493,7 @@
// not a space, if this char is the delimiter,
// then we have reached the end of
// the record
- if (chrArry[i] == delimiter.charAt(0)) {
+ if (chrArry[i] == delimiter) {
// processingMultiLine = false;
// fix put in, setting to false caused bug
// when processing multiple multi-line
@@ -412,7 +504,7 @@
qualiFound = false;
continue;
}
- } else if (chrArry[i] == qualifier.charAt(0)) {
+ } else if (chrArry[i] == qualifier) {
qualiFound = true;
}
}
Modified: trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java
===================================================================
--- trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java 2006-10-24 21:36:11 UTC (rev 118)
+++ trunk/PZFileReader/src/main/java/net/sf/pzfilereader/util/ParserUtils.java 2006-10-24 23:33:29 UTC (rev 119)
@@ -44,6 +44,17 @@
}
/**
+ * @deprecated should only use the splitLine with a CHAR.
+ * @param line
+ * @param delimiter
+ * @param qualifier
+ * @return
+ */
+ public static List splitLine(String line, final String delimiter, final String qualifier) {
+ return splitLine(line, delimiter != null ? delimiter.charAt(0) : 0, qualifier != null ? qualifier.charAt(0) : 0);
+ }
+
+ /**
* Returns an ArrayList of items in a delimited string. If there is no
* qualifier around the text, the qualifier parameter can be left null, or
* empty. There should not be any line breaks in the string. Each line of
@@ -57,7 +68,7 @@
* qualifier which is surrounding the text
* @return ArrayList
*/
- public static List splitLine(String line, final String delimiter, final String qualifier) {
+ public static List splitLine(String line, final char delimiter, final char qualifier) {
final ArrayList list = new ArrayList();
// String temp = "";
boolean beginQualifier = false;
@@ -73,12 +84,13 @@
// line which has not yet been read
// check to see if there is a text qualifier
final char currentChar = line.charAt(i);
- if (qualifier != null && qualifier.trim().length() > 0) {
- if (line.substring(i, i + 1).equals(qualifier) && !beginQualifier && !beginNoQualifier) {
+ final String currentString = String.valueOf(currentChar);
+ if (qualifier > 0) {
+ if (currentChar == qualifier && !beginQualifier && !beginNoQualifier) {
// begining of a set of data
beginQualifier = true;
- } else if (!beginQualifier && !beginNoQualifier && !line.substring(i, i + 1).equals(qualifier)
- && !lTrim(remainderOfLine).startsWith(qualifier)) {
+ } else if (!beginQualifier && !beginNoQualifier && currentChar != qualifier
+ && lTrim(remainderOfLine).charAt(0) != qualifier) {
// try to account for empty space before qualifier starts
// we have not yet begun a qualifier and the char we are on
// is NOT
@@ -87,18 +99,18 @@
// make sure that this is not just an empty column with no
// qualifiers. ie
// "data",,"data"
- if (line.substring(i, i + 1).equals(delimiter)) {
+ if (currentChar == delimiter) {
list.add(sb.toString());
sb.delete(0, sb.length());
beginNoQualifier = false;
continue;// grab the next char
}
sb.append(currentChar);
- } else if ((!beginNoQualifier) && line.substring(i, i + 1).equals(qualifier) && beginQualifier
- && (lTrim(line.substring(i + 1)).length() == 0
+ } else if (!beginNoQualifier && currentChar == qualifier && beginQualifier
+ && (i == line.length() - 1 || lTrim(remainderOfLine.substring(1)).length() == 0
// this will be true on empty undelmited columns at the
// end of theline
- || lTrimKeepTabs(line.substring(i + 1)).substring(0, 1).equals(delimiter))) {
+ || lTrimKeepTabs(remainderOfLine).charAt(1) == delimiter)) {
// end of a set of data that was qualified
list.add(sb.toString());
sb.delete(0, sb.length());
@@ -116,7 +128,7 @@
} else {
i += offset;
}
- } else if (beginNoQualifier && line.substring(i, i + 1).equals(delimiter)) {
+ } else if (beginNoQualifier && currentChar == delimiter) {
// check to see if we are done with an element that was not
// being qulified
list.add(sb.toString());
@@ -130,7 +142,7 @@
} else {
// not using a qualifier. Using a delimiter only
- if (line.substring(i, i + 1).equals(delimiter)) {
+ if (currentChar == delimiter) {
list.add(sb.toString());
sb.delete(0, sb.length());
} else {
@@ -139,17 +151,21 @@
}
}
+ // + this needs to be revisited...
+ String trimmed = sb.toString().trim();
// remove the ending text qualifier if needed
- if (qualifier != null && qualifier.trim().length() > 0 && sb.toString().trim().length() > 0) {
- if (sb.toString().trim().substring(sb.toString().trim().length() - 1).equals(qualifier)) {
- final String s = sb.toString().trim().substring(0, sb.toString().trim().length() - 1);
+ if (qualifier > 0 && trimmed.length() > 0) {
+ if (trimmed.charAt(trimmed.length() - 1) == qualifier) {
+ final String s = trimmed.substring(0, trimmed.length() - 1);
sb.delete(0, sb.length());
sb.append(s);
}
}
- if (qualifier == null || qualifier.trim().length() == 0 || beginQualifier || beginNoQualifier
- || line.trim().endsWith(delimiter)) {
+ String trimmed2 = line.trim();
+ int lengthLeft = trimmed2.length();
+ if (qualifier <= 0 || beginQualifier || beginNoQualifier || lengthLeft > 0
+ && trimmed2.charAt(lengthLeft - 1) == delimiter) {
// also account for a delimiter with an empty column at the end that
// was not qualified
// check to see if we need to add the last column in..this will
@@ -173,7 +189,7 @@
* @return int
*/
- public static int getDelimiterOffset(final String line, final int start, final String delimiter) {
+ public static int getDelimiterOffset(final String line, final int start, final char delimiter) {
int idx = line.indexOf(delimiter, start);
if (idx >= 0) {
// idx++;
@@ -338,7 +354,7 @@
continue;
}
- lineData = splitLine(line, delimiter, qualifier);
+ lineData = splitLine(line, delimiter.charAt(0), qualifier.charAt(0));
for (int i = 0; i < lineData.size(); i++) {
final ColumnMetaData cmd = new ColumnMetaData();
cmd.setColName((String) lineData.get(i));
@@ -375,7 +391,7 @@
* @exception Exception
* @return ArrayList - ColumnMetaData
*/
- public static Map getColumnMDFromFile(final String line, final String delimiter, final String qualifier) throws Exception {
+ public static Map getColumnMDFromFile(final String line, final char delimiter, final char qualifier) throws Exception {
List lineData = null;
final List results = new ArrayList();
final Map columnMD = new LinkedHashMap();
@@ -420,7 +436,7 @@
continue;
}
- lineData = splitLine(line, delimiter, qualifier);
+ lineData = splitLine(line, delimiter.charAt(0), qualifier.charAt(0));
for (int i = 0; i < lineData.size(); i++) {
final ColumnMetaData cmd = new ColumnMetaData();
cmd.setColName((String) lineData.get(i));
@@ -471,12 +487,28 @@
* @param qualifier -
* qualifier being used
* @return boolean
+ * @deprecated use the char version
*/
public static boolean isMultiLine(final char[] chrArry, final String delimiter, final String qualifier) {
+ return isMultiLine(chrArry, delimiter != null ? delimiter.charAt(0) : 0, qualifier != null ? qualifier.charAt(0) : 0);
+ }
+ /**
+ * Determines if the given line is the first part of a multiline record
+ *
+ * @param chrArry -
+ * char data of the line
+ * @param delimiter -
+ * delimiter being used
+ * @param qualifier -
+ * qualifier being used
+ * @return boolean
+ */
+ public static boolean isMultiLine(final char[] chrArry, final char delimiter, final char qualifier) {
+
// check if the last char is the qualifier, if so then this a good
// chance it is not multiline
- if (chrArry[chrArry.length - 1] != qualifier.charAt(0)) {
+ if (chrArry[chrArry.length - 1] != qualifier) {
// could be a potential line break
boolean qualiFound = false;
for (int i = chrArry.length - 1; i >= 0; i--) {
@@ -490,13 +522,13 @@
// not a space, if this char is the delimiter, then we
// have a line break
// in the record
- if (chrArry[i] == delimiter.charAt(0)) {
+ if (chrArry[i] == delimiter) {
return true;
}
qualiFound = false;
continue;
}
- } else if (chrArry[i] == delimiter.charAt(0)) {
+ } else if (chrArry[i] == delimiter) {
// if we have a delimiter followed by a qualifier, then we
// have moved on
// to a new element and this could not be multiline. start a
@@ -505,13 +537,13 @@
for (int j = i - 1; j >= 0; j--) {
if (chrArry[j] == ' ') {
continue;
- } else if (chrArry[j] == qualifier.charAt(0)) {
+ } else if (chrArry[j] == qualifier) {
return false;
}
break;
}
- } else if (chrArry[i] == qualifier.charAt(0)) {
+ } else if (chrArry[i] == qualifier) {
qualiFound = true;
}
}
@@ -529,7 +561,7 @@
// the delimiter and qualifier
continue;
}
- if (chrArry[i] == delimiter.charAt(0)) {
+ if (chrArry[i] == delimiter) {
return true;
}
break;
@@ -678,6 +710,7 @@
/**
* Use this method to find the index of a column.
+ *
* @author Benoit Xhenseval
* @param key
* @param columnMD
@@ -759,9 +792,9 @@
* </p>
*
* <pre>
- * StringUtils.padding(0, 'e') = ""
- * StringUtils.padding(3, 'e') = "eee"
- * StringUtils.padding(-2, 'e') = IndexOutOfBoundsException
+ * StringUtils.padding(0, 'e') = ""
+ * StringUtils.padding(3, 'e') = "eee"
+ * StringUtils.padding(-2, 'e') = IndexOutOfBoundsException
* </pre>
*
* <p>
@@ -794,6 +827,7 @@
/**
* Build a map of name/position based on a list of ColumnMetaData.
+ *
* @author Benoit Xhenseval
* @param columns
* @return a new Map
@@ -805,7 +839,8 @@
int idx = 0;
for (final Iterator it = columns.iterator(); it.hasNext(); idx++) {
final ColumnMetaData meta = (ColumnMetaData) it.next();
- //map.put(meta.getColName(), Integer.valueOf(idx)); breaks 1.4 compile
+ // map.put(meta.getColName(), Integer.valueOf(idx)); breaks 1.4
+ // compile
map.put(meta.getColName(), new Integer(idx));
}
}
Modified: trunk/PZFileReader/src/test/java/net/sf/pzfilereader/parserutils/ParserUtilsSplitLineTest.java
===================================================================
--- trunk/PZFileReader/src/test/java/net/sf/pzfilereader/parserutils/ParserUtilsSplitLineTest.java 2006-10-24 21:36:11 UTC (rev 118)
+++ trunk/PZFileReader/src/test/java/net/sf/pzfilereader/parserutils/ParserUtilsSplitLineTest.java 2006-10-24 23:33:29 UTC (rev 119)
@@ -7,127 +7,100 @@
import junit.framework.TestCase;
/**
- * Test the functionality of the splitLine method. This method returns
- * a List of Strings. Each element of the list represents a column created
- * by the parser from the delimited String.
+ * Test the functionality of the splitLine method. This method returns a List of
+ * Strings. Each element of the list represents a column created by the parser
+ * from the delimited String.
*
- * @author Paul Zepernick
+ * @author Paul Zepernick
*/
-public class ParserUtilsSplitLineTest extends TestCase{
- private static final String[] DELIMITED_DATA_NO_BREAKS = {"Column 1","Column 2", "Column 3", "Column 4", "Column 5"};
- private static final String[] DELIMITED_DATA_WITH_BREAKS = {"Column 1 \r\n\r\n Test After Break \r\n Another Break",
- "Column 2",
- "Column 3 \r\n\r\n Test After Break",
- "Column 4",
- "Column 5 \r\n\r\n Test After Break\r\n Another Break"};
- //TODO think of a situation that actually breaks the parse. This still works because of the way it is coded
- //to handle the excel CSV. Excel CSV has some elements qualified and others not
+public class ParserUtilsSplitLineTest extends TestCase {
+ private static final String[] DELIMITED_DATA_NO_BREAKS = { "Column 1", "Column 2", "Column 3", "Column 4", "Column 5" };
+
+ private static final String[] DELIMITED_DATA_WITH_BREAKS = { "Column 1 \r\n\r\n Test After Break \r\n Another Break",
+ "Column 2", "Column 3 \r\n\r\n Test After Break", "Column 4", "Column 5 \r\n\r\n Test After Break\r\n Another Break" };
+
+ // TODO think of a situation that actually breaks the parse. This still
+ // works because of the way it is coded
+ // to handle the excel CSV. Excel CSV has some elements qualified and others
+ // not
private static final String DELIMITED_BAD_DATA = "\"column 1\",\"column 2 ,\"column3\"";
-
- //0 = delimiter
- //1 = qualifier
- private static final String[][] DELIM_QUAL_PAIR = {
- {",","\""},
- {"\t","\""},
- {"|","\""},
- {"_","\""},
- {",",null},
- {"|",null},
- {"\t",null},
- };
-
+
+ // 0 = delimiter
+ // 1 = qualifier
+ private static final char[][] DELIM_QUAL_PAIR = { { ',', '\"' }, { '\t', '\"' }, { '|', '\"' }, { '_', '\"' }, { ',', 0 },
+ { '|', 0 }, { '\t', 0 } };
+
/**
* Test without any line breaks
- *
+ *
*/
- public void testNoLineBreaks(){
- //loop down all delimiter qualifier pairs to test
- for (int i = 0; i < DELIM_QUAL_PAIR.length; i++){
- final String d = DELIM_QUAL_PAIR[i][0];
- final String q = DELIM_QUAL_PAIR[i][1];
-
- final String txtToParse;
- if (q == null){
- txtToParse = UnitTestUtils.buildDelimString(DELIMITED_DATA_NO_BREAKS,
- d.charAt(0));
- }else{
- txtToParse = UnitTestUtils.buildDelimString(DELIMITED_DATA_NO_BREAKS,
- d.charAt(0),
- q.charAt(0));
- }
-
- final List splitLineResults = ParserUtils.splitLine(txtToParse.toString(),
- d, q);
-
-
- //check to make sure we have the same amount of elements which were expected
- assertEquals("Did Not Get Amount Of Elements Expected (d = " + d + " q = " + q + ")",
+ public void testNoLineBreaks() {
+ // loop down all delimiter qualifier pairs to test
+ for (int i = 0; i < DELIM_QUAL_PAIR.length; i++) {
+ final char d = DELIM_QUAL_PAIR[i][0];
+ final char q = DELIM_QUAL_PAIR[i][1];
+
+ final String txtToParse = UnitTestUtils.buildDelimString(DELIMITED_DATA_NO_BREAKS, d, q);
+
+ final List splitLineResults = ParserUtils.splitLine(txtToParse, d, q);
+
+ // check to make sure we have the same amount of elements which were
+ // expected
+ assertEquals("Did Not Get Amount Of Elements Expected (d = [" + d + "] q = [" + q + "] txt [" + txtToParse + "])",
DELIMITED_DATA_NO_BREAKS.length, splitLineResults.size());
-
- //loop through each value and compare what came back
- for (int j = 0 ; j < DELIMITED_DATA_NO_BREAKS.length; j ++){
- assertEquals("Data Element Value Does Not Match (d = " + d + " q = " + q + ")",
- DELIMITED_DATA_NO_BREAKS[j], (String)splitLineResults.get(j));
+
+ // loop through each value and compare what came back
+ for (int j = 0; j < DELIMITED_DATA_NO_BREAKS.length; j++) {
+ assertEquals("Data Element Value Does Not Match (d = [" + d + "] q = [" + q + "] txt [" + txtToParse + "])",
+ DELIMITED_DATA_NO_BREAKS[j], (String) splitLineResults.get(j));
}
}
-
-
+
}
-
+
/**
* Test without any line breaks
- *
+ *
*/
- public void testLineBreaks(){
- //loop down all delimiter qualifier pairs to test
- for (int i = 0; i < DELIM_QUAL_PAIR.length; i++){
- final String d = DELIM_QUAL_PAIR[i][0];
- final String q = DELIM_QUAL_PAIR[i][1];
-
- final String txtToParse;
- if (DELIM_QUAL_PAIR[i][1] == null){
- txtToParse = UnitTestUtils.buildDelimString(DELIMITED_DATA_WITH_BREAKS,
- DELIM_QUAL_PAIR[i][0].charAt(0));
- }else{
- txtToParse = UnitTestUtils.buildDelimString(DELIMITED_DATA_WITH_BREAKS,
- DELIM_QUAL_PAIR[i][0].charAt(0),
- DELIM_QUAL_PAIR[i][1].charAt(0));
- }
-
- final List splitLineResults = ParserUtils.splitLine(txtToParse.toString(),
- DELIM_QUAL_PAIR[i][0], DELIM_QUAL_PAIR[i][1]);
-
-
- //check to make sure we have the same amount of elements which were expected
+ public void testLineBreaks() {
+ // loop down all delimiter qualifier pairs to test
+ for (int i = 0; i < DELIM_QUAL_PAIR.length; i++) {
+ final char d = DELIM_QUAL_PAIR[i][0];
+ final char q = DELIM_QUAL_PAIR[i][1];
+
+ final String txtToParse = UnitTestUtils.buildDelimString(DELIMITED_DATA_WITH_BREAKS, d, q);
+
+ final List splitLineResults = ParserUtils.splitLine(txtToParse.toString(), d, q);
+
+ // check to make sure we have the same amount of elements which were
+ // expected
assertEquals("Did Not Get Amount Of Elements Expected (d = " + d + " q = " + q + ")",
DELIMITED_DATA_WITH_BREAKS.length, splitLineResults.size());
-
- //loop through each value and compare what came back
- for (int j = 0 ; j < DELIMITED_DATA_WITH_BREAKS.length; j ++){
- assertEquals("Data Element Value Does Not Match (d = " + d + " q = " + q + ")",
- DELIMITED_DATA_WITH_BREAKS[j], (String)splitLineResults.get(j));
+
+ // loop through each value and compare what came back
+ for (int j = 0; j < DELIMITED_DATA_WITH_BREAKS.length; j++) {
+ assertEquals("Data Element Value Does Not Match (d = " + d + " q = " + q + ")", DELIMITED_DATA_WITH_BREAKS[j],
+ (String) splitLineResults.get(j));
}
-
+
}
}
-
+
/**
- * Test to make sure we get the correct amount of elements for malformed data
+ * Test to make sure we get the correct amount of elements for malformed
+ * data
*
* @param args
*/
- public void testMalformedData(){
- final List splitLineResults = ParserUtils.splitLine(DELIMITED_BAD_DATA,
- ",", "\"");
-
-
- assertEquals("Expecting 2 Data Elements From The Malformed Data",
- 2, splitLineResults.size());
-
+ public void testMalformedData() {
+ final List splitLineResults = ParserUtils.splitLine(DELIMITED_BAD_DATA, ',', '\"');
+
+ assertEquals("Expecting 2 Data Elements From The Malformed Data", 2, splitLineResults.size());
+
}
-
+
public static void main(final String[] args) {
junit.textui.TestRunner.run(ParserUtilsSplitLineTest.class);
}
-
+
}
Modified: trunk/PZFileReader/src/test/java/net/sf/pzfilereader/utilities/UnitTestUtils.java
===================================================================
--- trunk/PZFileReader/src/test/java/net/sf/pzfilereader/utilities/UnitTestUtils.java 2006-10-24 21:36:11 UTC (rev 118)
+++ trunk/PZFileReader/src/test/java/net/sf/pzfilereader/utilities/UnitTestUtils.java 2006-10-24 23:33:29 UTC (rev 119)
@@ -6,46 +6,30 @@
* @author Paul Zepernick
*/
public class UnitTestUtils {
-
+
/**
* Builds a delimited qualified string containing the elements passed in
*
* @param elements
- * Elements To Include In Delimited String
+ * Elements To Include In Delimited String
* @param delimiter
* @param qualifier
* @return String
*/
- public static String buildDelimString(final String[] elements, final char delimiter, final char qualifier){
+ public static String buildDelimString(final String[] elements, final char delimiter, final char qualifier) {
final StringBuffer sb = new StringBuffer();
- for (int i = 0; i < elements.length; i++){
- if (i > 0){
+ for (int i = 0; i < elements.length; i++) {
+ if (i > 0) {
sb.append(delimiter);
}
- sb.append(qualifier).append(elements[i]).append(qualifier);
+ if (qualifier != 0) {
+ sb.append(qualifier);
+ }
+ sb.append(elements[i]);
+ if (qualifier != 0) {
+ sb.append(qualifier);
+ }
}
-
-
return sb.toString();
}
-
- /** Builds a delimited string containing the elements passed in
- *
- * @param elements
- * Elements To Include In Delimited String
- * @param delimiter
- * @return String
- */
- public static String buildDelimString(final String[] elements, final char delimiter){
- final StringBuffer sb = new StringBuffer();
- for (int i = 0; i < elements.length; i++){
- if (i > 0) {
- sb.append(delimiter);
- }
- sb.append(elements[i]);
- }
-
-
- return sb.toString();
- }
}
Property changes on: trunk/PZFileReaderSamples
___________________________________________________________________
Name: svn:ignore
- target
+ target
velocity.log
ydocErr.log
Modified: trunk/PZFileReaderSamples/qalab.xml
===================================================================
--- trunk/PZFileReaderSamples/qalab.xml 2006-10-24 21:36:11 UTC (rev 118)
+++ trunk/PZFileReaderSamples/qalab.xml 2006-10-24 23:33:29 UTC (rev 119)
@@ -29,6 +29,14 @@
<summaryresult date="2006-10-18" filecount="2" statvalue="2" type="pmd"/>
<summaryresult date="2006-10-18" filecount="6" statvalue="8" type="findbugs"/>
<summaryresult date="2006-10-18" filecount="6" statvalue="78" type="simian"/>
+ <summaryresult date="2006-10-20" filecount="34" statvalue="151" type="checkstyle"/>
+ <summaryresult date="2006-10-20" filecount="2" statvalue="2" type="pmd"/>
+ <summaryresult date="2006-10-20" filecount="6" statvalue="8" type="findbugs"/>
+ <summaryresult date="2006-10-20" filecount="6" statvalue="78" type="simian"/>
+ <summaryresult date="2006-10-25" filecount="34" statvalue="160" type="checkstyle"/>
+ <summaryresult date="2006-10-25" filecount="2" statvalue="2" type="pmd"/>
+ <summaryresult date="2006-10-25" filecount="6" statvalue="8" type="findbugs"/>
+ <summaryresult date="2006-10-25" filecount="6" statvalue="110" type="simian"/>
</summary>
<file id="com_pz_reader_examples_numericsanddates_package.html" path="com/pz/reader/examples/numericsanddates/package.html">
<result date="2006-10-16" statvalue="1" type="checkstyle"/>
@@ -159,83 +167,119 @@
<file id="net_sf_pzfilereader_examples_package.html" path="net/sf/pzfilereader/examples/package.html">
<result date="2006-10-17" statvalue="1" type="checkstyle"/>
<result date="2006-10-18" statvalue="1" type="checkstyle"/>
+ <result date="2006-10-20" statvalue="1" type="checkstyle"/>
+ <result date="2006-10-25" statvalue="1" type="checkstyle"/>
</file>
<file
id="net_sf_pzfilereader_examples_largedataset_largecsvperformancetest_package.html" path="net/sf/pzfilereader/examples/largedataset/largecsvperformancetest/package.html">
<result date="2006-10-17" statvalue="1" type="checkstyle"/>
<result date="2006-10-18" statvalue="1" type="checkstyle"/>
+ <result date="2006-10-20" statvalue="1" type="checkstyle"/>
+ <result date="2006-10-25" statvalue="1" type="checkstyle"/>
</file>
<file
id="net_sf_pzfilereader_examples_multilinedelimitedrecord_package.html" path="net/sf/pzfilereader/examples/multilinedelimitedrecord/package.html">
<result date="2006-10-17" statvalue="1" type="checkstyle"/>
<result date="2006-10-18" statvalue="1" type="checkstyle"/>
+ <result date="2006-10-20" statvalue="1" type="checkstyle"/>
+ <result date="2006-10-25" statvalue="1" type="checkstyle"/>
</file>
<file
id="net_sf_pzfilereader_examples_numericsanddates_package.html" path="net/sf/pzfilereader/examples/numericsanddates/package.html">
<result date="2006-10-17" statvalue="1" type="checkstyle"/>
<result date="2006-10-18" statvalue="1" type="checkstyle"/>
+ <result date="2006-10-20" statvalue="1" type="checkstyle"/>
+ <result date="2006-10-25" statvalue="1" type="checkstyle"/>
</file>
<file
id="net_sf_pzfilereader_examples_delimitedcolumnnamesinfile_package.html" path="net/sf/pzfilereader/examples/delimitedcolumnnamesinfile/package.html">
<result date="2006-10-17" statvalue="1" type="checkstyle"/>
<result date="2006-10-18" statvalue="1" type="checkstyle"/>
+ <result date="2006-10-20" statvalue="1" type="checkstyle"/>
+ <result date="2006-10-25" statvalue="1" type="checkstyle"/>
</file>
<file
id="net_sf_pzfilereader_examples_delimiteddynamiccolumns_package.html" path="net/sf/pzfilereader/examples/delimiteddynamiccolumns/package.html">
<result date="2006-10-17" statvalue="1" type="checkstyle"/>
<result date="2006-10-18" statvalue="1" type="checkstyle"/>
+ <result date="2006-10-20" statvalue="1" type="checkstyle"/>
+ <result date="2006-10-25" statvalue="1" type="checkstyle"/>
</file>
<file id="net_sf_pzfilereader_examples_exporttoexcel_package.html" path="net/sf/pzfilereader/examples/exporttoexcel/package.html">
<result date="2006-10-17" statvalue="1" type="checkstyle"/>
<result date="2006-10-18" statvalue="1" type="checkstyle"/>
+ <result date="2006-10-20" statvalue="1" type="checkstyle"/>
+ <result date="2006-10-25" statvalue="1" type="checkstyle"/>
</file>
<file
id="net_sf_pzfilereader_examples_largedataset_fixedlengthdynamiccolumns_package.html" path="net/sf/pzfilereader/examples/largedataset/fixedlengthdynamiccolumns/package.html">
<result date="2006-10-17" statvalue="1" type="checkstyle"/>
<result date="2006-10-18" statvalue="1" type="checkstyle"/>
+ <result date="2006-10-20" statvalue="1" type="checkstyle"/>
+ <result date="2006-10-25" statvalue="1" type="checkstyle"/>
</file>
<file id="net_sf_pzfilereader_examples_createsamplecsv_package.html" path="net/sf/pzfilereader/exampl...
[truncated message content] |
|
From: <be...@us...> - 2006-10-24 21:36:15
|
Revision: 118
http://svn.sourceforge.net/pzfilereader/?rev=118&view=rev
Author: benoitx
Date: 2006-10-24 14:36:11 -0700 (Tue, 24 Oct 2006)
Log Message:
-----------
Forgot to append the actual element.
Modified Paths:
--------------
trunk/PZFileReader/src/test/java/net/sf/pzfilereader/utilities/UnitTestUtils.java
Modified: trunk/PZFileReader/src/test/java/net/sf/pzfilereader/utilities/UnitTestUtils.java
===================================================================
--- trunk/PZFileReader/src/test/java/net/sf/pzfilereader/utilities/UnitTestUtils.java 2006-10-24 17:33:24 UTC (rev 117)
+++ trunk/PZFileReader/src/test/java/net/sf/pzfilereader/utilities/UnitTestUtils.java 2006-10-24 21:36:11 UTC (rev 118)
@@ -39,9 +39,10 @@
public static String buildDelimString(final String[] elements, final char delimiter){
final StringBuffer sb = new StringBuffer();
for (int i = 0; i < elements.length; i++){
- if (i > 0){
+ if (i > 0) {
sb.append(delimiter);
}
+ sb.append(elements[i]);
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <zep...@us...> - 2006-10-24 17:33:30
|
Revision: 117
http://svn.sourceforge.net/pzfilereader/?rev=117&view=rev
Author: zepernick
Date: 2006-10-24 10:33:24 -0700 (Tue, 24 Oct 2006)
Log Message:
-----------
expanded upon the tests. Made a little more generic. There is an
array of delimiters and qualifiers which we can fill in for whatever
we want to test. Implemented Benoit's formating suggestions.
Modified Paths:
--------------
trunk/PZFileReader/src/test/java/net/sf/pzfilereader/parserutils/ParserUtilsSplitLineTest.java
Added Paths:
-----------
trunk/PZFileReader/src/test/java/net/sf/pzfilereader/utilities/UnitTestUtils.java
Modified: trunk/PZFileReader/src/test/java/net/sf/pzfilereader/parserutils/ParserUtilsSplitLineTest.java
===================================================================
--- trunk/PZFileReader/src/test/java/net/sf/pzfilereader/parserutils/ParserUtilsSplitLineTest.java 2006-10-24 17:33:17 UTC (rev 116)
+++ trunk/PZFileReader/src/test/java/net/sf/pzfilereader/parserutils/ParserUtilsSplitLineTest.java 2006-10-24 17:33:24 UTC (rev 117)
@@ -3,6 +3,7 @@
import java.util.List;
import net.sf.pzfilereader.util.ParserUtils;
+import net.sf.pzfilereader.utilities.UnitTestUtils;
import junit.framework.TestCase;
/**
@@ -10,40 +11,119 @@
* a List of Strings. Each element of the list represents a column created
* by the parser from the delimited String.
*
- * @author zepernick
+ * @author Paul Zepernick
*/
public class ParserUtilsSplitLineTest extends TestCase{
- final String[] delimitedDataNoBreaks = {"Column 1","Column 2", "Column 3", "Column 4", "Column 5"};
- final String[] delimitedDataWithBreaks = {"Column 1","Column 2", "Column 3", "Column 4", "Column 5"};
+ private static final String[] DELIMITED_DATA_NO_BREAKS = {"Column 1","Column 2", "Column 3", "Column 4", "Column 5"};
+ private static final String[] DELIMITED_DATA_WITH_BREAKS = {"Column 1 \r\n\r\n Test After Break \r\n Another Break",
+ "Column 2",
+ "Column 3 \r\n\r\n Test After Break",
+ "Column 4",
+ "Column 5 \r\n\r\n Test After Break\r\n Another Break"};
+ //TODO think of a situation that actually breaks the parse. This still works because of the way it is coded
+ //to handle the excel CSV. Excel CSV has some elements qualified and others not
+ private static final String DELIMITED_BAD_DATA = "\"column 1\",\"column 2 ,\"column3\"";
+ //0 = delimiter
+ //1 = qualifier
+ private static final String[][] DELIM_QUAL_PAIR = {
+ {",","\""},
+ {"\t","\""},
+ {"|","\""},
+ {"_","\""},
+ {",",null},
+ {"|",null},
+ {"\t",null},
+ };
+
/**
- * Test CSV without any line breaks
+ * Test without any line breaks
*
*/
- public void testCSVNoLineBreaks(){
-
- final String delimiter = ",";
- final String qualifier = "\"";
- final StringBuffer txtToParse = new StringBuffer();
- for (int i = 0; i < delimitedDataNoBreaks.length; i++){
- if (i > 0){
- txtToParse.append(delimiter);
+ public void testNoLineBreaks(){
+ //loop down all delimiter qualifier pairs to test
+ for (int i = 0; i < DELIM_QUAL_PAIR.length; i++){
+ final String d = DELIM_QUAL_PAIR[i][0];
+ final String q = DELIM_QUAL_PAIR[i][1];
+
+ final String txtToParse;
+ if (q == null){
+ txtToParse = UnitTestUtils.buildDelimString(DELIMITED_DATA_NO_BREAKS,
+ d.charAt(0));
+ }else{
+ txtToParse = UnitTestUtils.buildDelimString(DELIMITED_DATA_NO_BREAKS,
+ d.charAt(0),
+ q.charAt(0));
}
- txtToParse.append(qualifier + delimitedDataNoBreaks[i] + qualifier);
+
+ final List splitLineResults = ParserUtils.splitLine(txtToParse.toString(),
+ d, q);
+
+
+ //check to make sure we have the same amount of elements which were expected
+ assertEquals("Did Not Get Amount Of Elements Expected (d = " + d + " q = " + q + ")",
+ DELIMITED_DATA_NO_BREAKS.length, splitLineResults.size());
+
+ //loop through each value and compare what came back
+ for (int j = 0 ; j < DELIMITED_DATA_NO_BREAKS.length; j ++){
+ assertEquals("Data Element Value Does Not Match (d = " + d + " q = " + q + ")",
+ DELIMITED_DATA_NO_BREAKS[j], (String)splitLineResults.get(j));
+ }
}
- List splitLineResults = ParserUtils.splitLine(txtToParse.toString(), delimiter, qualifier);
+ }
+
+ /**
+ * Test without any line breaks
+ *
+ */
+ public void testLineBreaks(){
+ //loop down all delimiter qualifier pairs to test
+ for (int i = 0; i < DELIM_QUAL_PAIR.length; i++){
+ final String d = DELIM_QUAL_PAIR[i][0];
+ final String q = DELIM_QUAL_PAIR[i][1];
+
+ final String txtToParse;
+ if (DELIM_QUAL_PAIR[i][1] == null){
+ txtToParse = UnitTestUtils.buildDelimString(DELIMITED_DATA_WITH_BREAKS,
+ DELIM_QUAL_PAIR[i][0].charAt(0));
+ }else{
+ txtToParse = UnitTestUtils.buildDelimString(DELIMITED_DATA_WITH_BREAKS,
+ DELIM_QUAL_PAIR[i][0].charAt(0),
+ DELIM_QUAL_PAIR[i][1].charAt(0));
+ }
- //check to make sure we have the same amount of elements which were expected
- assertEquals(delimitedDataNoBreaks.length, splitLineResults.size());
-
- //loop through each value and compare what came back
- for (int i = 0 ; i < delimitedDataNoBreaks.length; i ++){
- assertEquals(delimitedDataNoBreaks[i], (String)splitLineResults.get(i));
+ final List splitLineResults = ParserUtils.splitLine(txtToParse.toString(),
+ DELIM_QUAL_PAIR[i][0], DELIM_QUAL_PAIR[i][1]);
+
+
+ //check to make sure we have the same amount of elements which were expected
+ assertEquals("Did Not Get Amount Of Elements Expected (d = " + d + " q = " + q + ")",
+ DELIMITED_DATA_WITH_BREAKS.length, splitLineResults.size());
+
+ //loop through each value and compare what came back
+ for (int j = 0 ; j < DELIMITED_DATA_WITH_BREAKS.length; j ++){
+ assertEquals("Data Element Value Does Not Match (d = " + d + " q = " + q + ")",
+ DELIMITED_DATA_WITH_BREAKS[j], (String)splitLineResults.get(j));
+ }
+
}
+ }
+
+ /**
+ * Test to make sure we get the correct amount of elements for malformed data
+ *
+ * @param args
+ */
+ public void testMalformedData(){
+ final List splitLineResults = ParserUtils.splitLine(DELIMITED_BAD_DATA,
+ ",", "\"");
+ assertEquals("Expecting 2 Data Elements From The Malformed Data",
+ 2, splitLineResults.size());
+
}
public static void main(final String[] args) {
Added: trunk/PZFileReader/src/test/java/net/sf/pzfilereader/utilities/UnitTestUtils.java
===================================================================
--- trunk/PZFileReader/src/test/java/net/sf/pzfilereader/utilities/UnitTestUtils.java (rev 0)
+++ trunk/PZFileReader/src/test/java/net/sf/pzfilereader/utilities/UnitTestUtils.java 2006-10-24 17:33:24 UTC (rev 117)
@@ -0,0 +1,50 @@
+package net.sf.pzfilereader.utilities;
+
+/**
+ * Static utility methods for Junit tests
+ *
+ * @author Paul Zepernick
+ */
+public class UnitTestUtils {
+
+ /**
+ * Builds a delimited qualified string containing the elements passed in
+ *
+ * @param elements
+ * Elements To Include In Delimited String
+ * @param delimiter
+ * @param qualifier
+ * @return String
+ */
+ public static String buildDelimString(final String[] elements, final char delimiter, final char qualifier){
+ final StringBuffer sb = new StringBuffer();
+ for (int i = 0; i < elements.length; i++){
+ if (i > 0){
+ sb.append(delimiter);
+ }
+ sb.append(qualifier).append(elements[i]).append(qualifier);
+ }
+
+
+ return sb.toString();
+ }
+
+ /** Builds a delimited string containing the elements passed in
+ *
+ * @param elements
+ * Elements To Include In Delimited String
+ * @param delimiter
+ * @return String
+ */
+ public static String buildDelimString(final String[] elements, final char delimiter){
+ final StringBuffer sb = new StringBuffer();
+ for (int i = 0; i < elements.length; i++){
+ if (i > 0){
+ sb.append(delimiter);
+ }
+ }
+
+
+ return sb.toString();
+ }
+}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <zep...@us...> - 2006-10-24 17:33:24
|
Revision: 116
http://svn.sourceforge.net/pzfilereader/?rev=116&view=rev
Author: zepernick
Date: 2006-10-24 10:33:17 -0700 (Tue, 24 Oct 2006)
Log Message:
-----------
expanded upon the tests. Made a little more generic. There is an
array of delimiters and qualifiers which we can fill in for whatever
we want to test. Implemented Benoit's formating suggestions.
Added Paths:
-----------
trunk/PZFileReader/src/test/java/net/sf/pzfilereader/utilities/
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
