From: <res...@us...> - 2010-09-24 18:26:15
|
Revision: 3628 http://bigdata.svn.sourceforge.net/bigdata/?rev=3628&view=rev Author: resendes Date: 2010-09-24 18:26:07 +0000 (Fri, 24 Sep 2010) Log Message: ----------- - Continued clean-up of com.bigdata.util.* - Removed unreferenced/unused classes - Added tests and support classes for CSVReader Modified Paths: -------------- branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/CSVReader.java branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/TestAll.java branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/TestCSVReader.java Added Paths: ----------- branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/CSVReaderBuilder.java Removed Paths: ------------- branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/ByteBufferBitVector.java branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/ChecksumError.java branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/ClassLoaderUtil.java branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/TestByteBufferBitVector.java Deleted: branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/ByteBufferBitVector.java =================================================================== --- branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/ByteBufferBitVector.java 2010-09-24 17:46:05 UTC (rev 3627) +++ branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/ByteBufferBitVector.java 2010-09-24 18:26:07 UTC (rev 3628) @@ -1,167 +0,0 @@ -/* - -Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved. - -Contact: - SYSTAP, LLC - 4501 Tower Road - Greensboro, NC 27410 - lic...@bi... - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ -/* - * Created on Aug 6, 2009 - */ - -package com.bigdata.util; - -import it.unimi.dsi.bits.AbstractBitVector; - -import java.nio.ByteBuffer; - -import cern.colt.bitvector.BitVector; - -/** - * Wraps a {@link ByteBuffer} as a read-only {@link BitVector}. - * - * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ - */ -public class ByteBufferBitVector extends AbstractBitVector { - - /** - * The {@link ByteBuffer} containing the backing data. - */ - final private ByteBuffer b; - - /** - * The #of bits in the vector. - */ - private final long len; - - /** - * The bit offset into the {@link ByteBuffer} of the first bit in the - * vector. - */ - private final long off; - - final public long length() { - - return len; - - } - - /** - * Ctor assumes that all bits in the buffer are used. - * - * @param b - * The buffer. - */ - public ByteBufferBitVector(final ByteBuffer b) { - - this(b, 0/* offset */, b == null ? 0 : b.capacity() * 8/* len */); - - } - - /** - * - * @param b - * The buffer. - * @param off - * The offset from the start of the buffer for the view. - * @param len - * The #of bits which will be included in the view. - */ - public ByteBufferBitVector(final ByteBuffer b, final long off, - final long len) { - - if (b == null) - throw new IllegalArgumentException(); - - if (len < 0) - throw new IllegalArgumentException(); - - if (len < 0) - throw new IllegalArgumentException(); - - if (off + len > b.capacity() * 8L) - throw new IllegalArgumentException(); - - this.b = b; - - this.len = len; - - this.off = off; - - } - - /** - * Return the index of the byte in which the bit with the given index is - * encoded. - * - * @param bitIndex - * The bit index. - * - * @return The byte index. - */ - final protected int byteIndexForBit(final long bitIndex) { - - return ((int) ((bitIndex + off) / 8)); - - } - - /** - * Return the offset within the byte in which the bit is coded of the bit - * (this is just the remainder <code>bitIndex % 8</code>). - * - * @param bitIndex - * The bit index into the byte[]. - * - * @return The offset of the bit in the appropriate byte. - */ - final protected int withinByteIndexForBit(final long bitIndex) { - - return (int) ((bitIndex + off) % 8); - - } - - /** - * Extract and return a bit coded flag. - * - * @param offset - * The offset in the buffer of the start of the byte[] sequence - * in which the bit coded flags are stored. - * @param index - * The index of the bit. - * - * @return The value of the bit. - */ - public boolean getBoolean(final long index) { - - if (index < 0 || index >= len) - throw new IndexOutOfBoundsException(); - - return (b.get(byteIndexForBit(index)) & (1 << withinByteIndexForBit(index))) != 0; - - } - -// // @todo override for mutation. -// public boolean set(final long index, final boolean value) { -// -// throw new UnsupportedOperationException(); -// -// } - -} Modified: branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/CSVReader.java =================================================================== --- branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/CSVReader.java 2010-09-24 17:46:05 UTC (rev 3627) +++ branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/CSVReader.java 2010-09-24 18:26:07 UTC (rev 3628) @@ -93,10 +93,8 @@ private final String name; - public String getName() { - - return name; - + public String getName() { + return name; } /** @@ -227,25 +225,29 @@ } /** - * Equal if the headers have the same data. + * Equal if the headers have the same name. */ - public boolean equals(Header o) { + @Override + public boolean equals(Object o) { + if (!(o instanceof Header)) + return false; + + Header h = (Header)o; - if(this==o) return true; - - return name.equals(o.name); - + return name.equals(h.name); } /** * Based on the header name. */ + @Override public int hashCode() { return name.hashCode(); } + @Override public String toString() { return name; @@ -324,54 +326,54 @@ } - public boolean setSkipCommentLines(boolean skipCommentLines) { +// public boolean setSkipCommentLines(boolean skipCommentLines) { +// +// boolean tmp = this.skipCommentLines; +// +// this.skipCommentLines = skipCommentLines; +// +// return tmp; +// +// } +// +// public boolean getSkipCommentLines() { +// +// return skipCommentLines; +// +// } - boolean tmp = this.skipCommentLines; +// public boolean setSkipBlankLines(boolean skipBlankLines) { +// +// boolean tmp = this.skipBlankLines; +// +// this.skipBlankLines = skipBlankLines; +// +// return tmp; +// +// } +// +// public boolean getSkipBlankLines() { +// +// return skipBlankLines; +// +// } - this.skipCommentLines = skipCommentLines; +// public boolean setTrimWhitespace(boolean trimWhitespace) { +// +// boolean tmp = this.trimWhitespace; +// +// this.trimWhitespace = trimWhitespace; +// +// return tmp; +// +// } +// +// public boolean getTrimWhitespace() { +// +// return trimWhitespace; +// +// } - return tmp; - - } - - public boolean getSkipCommentLines() { - - return skipCommentLines; - - } - - public boolean setSkipBlankLines(boolean skipBlankLines) { - - boolean tmp = this.skipBlankLines; - - this.skipBlankLines = skipBlankLines; - - return tmp; - - } - - public boolean getSkipBlankLines() { - - return skipBlankLines; - - } - - public boolean setTrimWhitespace(boolean trimWhitespace) { - - boolean tmp = this.trimWhitespace; - - this.trimWhitespace = trimWhitespace; - - return tmp; - - } - - public boolean getTrimWhitespace() { - - return trimWhitespace; - - } - /** * The #of milliseconds that the {@link CSVReader} should wait before * attempting to read another line from the source (when reading from @@ -505,8 +507,7 @@ } /** - * Trim whitespace and optional quotes from each value iff - * {@link #getTrimWhitespace()} is true. + * Trim whitespace and optional quotes. * * @param cols * The column values. @@ -651,7 +652,7 @@ */ public Header[] getHeaders() { - return headers.clone(); + return ((headers==null)? null : headers.clone()); } @@ -679,7 +680,7 @@ */ public void setHeader(int index,Header header) { - if (index < 0 || index > headers.length) + if (index < 0 || index >= headers.length) throw new IndexOutOfBoundsException(); if (header == null) @@ -688,7 +689,7 @@ headers[index] = header; } - + /** * Unsupported operation. */ Deleted: branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/ChecksumError.java =================================================================== --- branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/ChecksumError.java 2010-09-24 17:46:05 UTC (rev 3627) +++ branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/ChecksumError.java 2010-09-24 18:26:07 UTC (rev 3628) @@ -1,46 +0,0 @@ -/** - -Copyright (C) SYSTAP, LLC 2006-2007. All rights reserved. - -Contact: - SYSTAP, LLC - 4501 Tower Road - Greensboro, NC 27410 - lic...@bi... - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ -/* - * Created on Nov 5, 2006 - */ - -package com.bigdata.util; - -/** - * Exception thrown when the checksum field does not match the checksum computed - * for the data being read. This is a serious error and indicates bad logic - * and/or corrupt data. - * - * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ - */ -public class ChecksumError extends RuntimeException { - - private static final long serialVersionUID = -9067118459184074756L; - - public ChecksumError(String msg) { - super( msg ); - } - -} \ No newline at end of file Deleted: branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/ClassLoaderUtil.java =================================================================== --- branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/ClassLoaderUtil.java 2010-09-24 17:46:05 UTC (rev 3627) +++ branches/bbb_cleanup/bigdata-core/src/main/java/com/bigdata/util/ClassLoaderUtil.java 2010-09-24 18:26:07 UTC (rev 3628) @@ -1,328 +0,0 @@ -/* - -Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved. - -Contact: - SYSTAP, LLC - 4501 Tower Road - Greensboro, NC 27410 - lic...@bi... - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -*/ - -package com.bigdata.util; - -import com.bigdata.util.config.LogUtil; - -import org.apache.log4j.Level; -import org.apache.log4j.Logger; - -import java.io.File; -import java.io.IOException; -import java.io.Serializable; - -import java.lang.reflect.Method; - -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; - -import java.rmi.Remote; -import java.rmi.RemoteException; -import java.rmi.activation.ActivationException; -import java.rmi.activation.ActivationID; - -import java.security.SecureClassLoader; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.StringTokenizer; - -import net.jini.io.MarshalledInstance; -import net.jini.loader.ClassAnnotation; - -/** - * This class provides useful utilities for creating and manipulating - * class loaders. Although it can be used for other purposes, it is - * generally intended for debugging. - */ -public class ClassLoaderUtil { - - /** Configure logger */ - private static Logger logger = - LogUtil.getLog4jLogger(ClassLoaderUtil.class); - - // Private constructor to prevent instantiation - private ClassLoaderUtil() { } - - /** - * Utility method that converts the components of a <code>String</code> - * representing a classpath into file <code>URL</code>(s). - * - * @param classpath <code>String</code> containing components separated - * by path separators that represent the components - * making up a classpath - * - * @return a <code>URL[]</code> where - * each element of the array corresponds to one of the components - * in the <code>classpath</code> parameter. The path components - * are (potentially) expanded via - * <code>File.getCanonicalFile()</code> before converting to a - * <code>URL</code> format. - * - * @throws java.net.MalformedURLException - * If the path cannot be parsed as a URL - * @throws java.net.IOException - * If an I/O error occurs, - * which is possible because the construction of - * the canonical pathname may require filesystem queries - */ - public static URL[] getClasspathURLs(String classpath) - throws IOException, MalformedURLException - { - StringTokenizer st = new StringTokenizer(classpath,File.pathSeparator); - URL[] urls = new URL[st.countTokens()]; - for (int i=0; st.hasMoreTokens(); i++) { - urls[i] = - new File(st.nextToken()).getCanonicalFile().toURI().toURL(); - } - return urls; - } - - /** - * Utility method that converts the components of a <code>String</code> - * representing a codebase into standard <code>URL</code>(s). - * - * @param codebase <code>String</code> containing components separated - * by spaces in which each component is in - * <code>URL</code> format. - * - * @return a <code>URL[]</code> where - * each element of the array corresponds to one of the components - * in the <code>codebase</code> parameter - * - * @throws java.net.MalformedURLException - */ - public static URL[] getCodebaseURLs(String codebase) - throws MalformedURLException - { - StringTokenizer st = new StringTokenizer(codebase); - URL[] urls = new URL[st.countTokens()]; - for (int i=0; st.hasMoreTokens(); i++) { - urls[i] = new URL(st.nextToken()); - } - return urls; - } - - /** - * Utility method that converts the components of a <code>String</code> - * representing a codebase or classpath into <code>URL</code>(s). - * - * @param importCodebase <code>String</code> assumed (in order) to be - * either - * 1) a space delimited set of <code>URL</code>(s) - * representing a codebase or - * 2) a <code>File.pathSeparator</code> delimited set - * of class paths. - * - * @return a <code>URL[]</code> where - * each element of the array corresponds to one of the components - * in the <code>importCodebase</code> parameter - * - * @throws java.net.MalformedURLException - * If the path cannot be parsed as a URL - * @throws java.net.IOException - * If an I/O error occurs, - * which is possible because the construction of - * the canonical pathname may require filesystem queries - */ - public static URL[] getImportCodebaseURLs(String importCodebase) - throws IOException, MalformedURLException - { - try { - return getCodebaseURLs(importCodebase); - } catch (MalformedURLException me) { - return getClasspathURLs(importCodebase); - } - } - - /** - * Utility method that retrieves the components making up the class loader - * delegation tree for the current context class loader and returns each - * in an <code>ArrayList</code>. - * - * @return an <code>ArrayList</code> instance in which each element of the - * list is one of the components making up the current delegation - * tree. - */ - private static ArrayList getContextClassLoaderTree() { - Thread curThread = Thread.currentThread(); - ClassLoader curClassLoader = curThread.getContextClassLoader(); - return getClassLoaderTree(curClassLoader); - } - - /** - * Utility method that retrieves the components making up the class loader - * delegation tree for the given <code>classloader</code> parameter and - * returns them via an <code>ArrayList</code>. - * - * @param classloader <code>ClassLoader</code> instance whose delegation - * tree is to be retrieved and returned - * - * @return an <code>ArrayList</code> instance in which each element of the - * list is one of the components making up the delegation tree - * of the given class loader. - */ - private static ArrayList getClassLoaderTree(ClassLoader classloader) { - ArrayList loaderList = new ArrayList(); - while(classloader != null) { - loaderList.add(classloader); - classloader = classloader.getParent(); - } - loaderList.add(null); //Append boot classloader - Collections.reverse(loaderList); - return loaderList; - } - - /** - * Utility method that displays the class loader delegation tree for - * the current context class loader. For each class loader in the tree, - * this method displays the locations from which that class loader - * will retrieve and load requested classes. - * <p> - * This method can be useful when debugging problems related to the - * receipt of exceptions such as <code>ClassNotFoundException</code>. - */ - public static void displayContextClassLoaderTree() { - Thread curThread = Thread.currentThread(); - ClassLoader curClassLoader = curThread.getContextClassLoader(); - displayClassLoaderTree(curClassLoader); - } - - /** - * Utility method that displays the class loader delegation tree for - * the given class loader. For each class loader in the tree, this - * method displays the locations from which that class loader will - * retrieve and load requested classes. - * <p> - * This method can be useful when debugging problems related to the - * receipt of exceptions such as <code>ClassNotFoundException</code>. - * - * Note that although this class' logger level is used to determine - * whether or not to display any information at all, the output is - * actually displayed using System.out.println. This is done to - * produce more readable output than the logger might produce. - * - * @param description descriptive <code>String</code> that, if - * non-<code>null</code>, will be logged prior to - * displaying the information about the - * <code>classloader</code>. - * - * @param classloader <code>ClassLoader</code> instance whose delegation - * tree is to be displayed. - */ - public static void displayClassLoaderTree(ClassLoader classloader) { - displayClassLoaderTree(null, classloader); - } - - public static void displayClassLoaderTree(String description, - ClassLoader classloader) - { - if( logger.isEnabledFor(Level.DEBUG) ) { - if(description != null) { - logger.log(Level.DEBUG, description); - } - - ArrayList loaderList = getClassLoaderTree(classloader); - System.out.println("ClassLoader Tree has " - + loaderList.size() + " levels"); - System.out.println(" cl0 -- Boot ClassLoader "); - ClassLoader curClassLoader = null; - for(int i=1; i < loaderList.size(); i++) { - System.out.println(" |"); - curClassLoader = (ClassLoader)loaderList.get(i); - System.out.print(" cl"+i+" -- ClassLoader " - +curClassLoader+": "); - if(curClassLoader instanceof URLClassLoader) { - URL[] urls = ((URLClassLoader)(curClassLoader)).getURLs(); - if(urls != null) { - System.out.print(urls[0]); - for(int j=1;j<urls.length;j++){ - System.out.print(", "+urls[j]); - } - } else {//urls == null - System.out.print("null search path"); - } - } else { - if(curClassLoader instanceof SecureClassLoader) { - System.out.print("is instance of SecureClassLoader"); - } else { - System.out.print("is unknown ClassLoader type"); - } - } - System.out.println(""); - } - System.out.println(""); - } - } - - /** - * Handles a <i>class loader mismatch</i> between the given - * <code>Serializable</code> object and the given <code>Class</code> - * type. - * - * If the class name of the given <code>obj</code> parameter - * is the same as the name of <code>classType</code>, but - * <code>obj</code> is not an instance of <code>classType</code>, - * then the difference may be due to unequal class loaders for the - * two parameters; which is referred to as a <i>class loader mismatch</i>. - * When such a mismatch occurs, the <code>instanceof</code> operator - * will return <code>false</code> and attempts to cast the given - * <code>obj</code> to the given <code>classType</code> will result - * in a <code>ClassCastException</code>. - * - * To address the situation just described, this method attempts to - * "reload" the given <code>obj</code>, using the <code>ClassLoader</code> - * of <code>classType</code>. This is accomplished by first - * marshalling and then unmarshalling the given <code>obj</code>, - * while the <i>current context class loader</i> is set to the - * <code>ClassLoader</code> of <code>classType</code>. - * - * Upon success, the newly loaded object is returned; which can then - * be successfully cast to the given <code>classType</code>. If the - * reload operation fails, <code>null</code> is returned. - */ - public static Serializable instanceOf(Serializable obj, Class classType) { - - if( classType.isInstance(obj) ) return obj; - - Class objClass = obj.getClass(); - ClassLoader classTypeCl = classType.getClassLoader(); - //marshall-and-unmarshal using the class type's classloader - try { - MarshalledInstance mInst = new MarshalledInstance(obj); - Serializable newObj = - (Serializable)(mInst.get(classTypeCl, false, null, null)); - if( classType.isInstance(newObj) ) return newObj; - - } catch(Throwable t) { - return null; - } - return null; - } - -} Added: branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/CSVReaderBuilder.java =================================================================== --- branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/CSVReaderBuilder.java (rev 0) +++ branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/CSVReaderBuilder.java 2010-09-24 18:26:07 UTC (rev 3628) @@ -0,0 +1,204 @@ +package com.bigdata.util; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; + +public class CSVReaderBuilder { + /** <code>List</code> holding the header row, if any. */ + private final List<String> header = new ArrayList<String>(); + /** <code>List</code> of <code>List</code>s holding the columnar data */ + private final List<List<String>>rows = new ArrayList<List<String>>(); + /** <code>List</code> holding the current row being operated on. */ + private final List<String> currentRow = new ArrayList<String>(); + /** Flag determining whether or not to display the header row. Default is false. */ + private boolean suppressHeader = false; + /** Flag determining whether or not to display quoted content. Default is false. */ + private boolean suppressQoutes = false; + /** <code>String</code> used for column delimiter. Default is a comma. */ + private String columnDelimiter = ","; + /** <code>String</code> used for row delimiter. Default is a newline. */ + private String rowDelimter = "\n"; + /** <code>String</code> used for quote delimiter. Default is a double quote. */ + private String quoteDelimter = "\""; + + //Setters and getters + public String getQuoteDelimter() { + return quoteDelimter; + } + public void setQuoteDelimter(String quoteDelimter) { + this.quoteDelimter = quoteDelimter; + } + public boolean isSuppressQoutes() { + return suppressQoutes; + } + public void setSuppressQoutes(boolean suppressQoutes) { + this.suppressQoutes = suppressQoutes; + } + public String getColumnDelimiter() { + return columnDelimiter; + } + public void setColumnDelimiter(String columnDelimiter) { + this.columnDelimiter = columnDelimiter; + } + public String getRowDelimiter() { + return rowDelimter; + } + public void setRowDelimiter(String rowDelimter) { + this.rowDelimter = rowDelimter; + } + public boolean isSuppressHeader() { + return suppressHeader; + } + public void setSuppressHeader(boolean suppressHeader) { + this.suppressHeader = suppressHeader; + } + + /** Default constructor */ + CSVReaderBuilder() {} + + /** Adds the given string to the header row */ + public CSVReaderBuilder header(String h){header.add(h); return this;} + + /** Adds the given string to the current row */ + public CSVReaderBuilder column(String c){currentRow.add(c); return this;} + + /** Creates a new row by: 1) flushing the current row data, if any, to the collection + * of row data and 2) clearing the the current row. + */ + public CSVReaderBuilder newRow() { + flushCurrentRow(); + currentRow.clear(); + return this; + } + + /** + * Helper method for flushing and clearing the current row. + */ + private void flushCurrentRow() { + if (!currentRow.isEmpty()) { + List<String> t = new ArrayList<String>(); + for (String s: currentRow) { + t.add(s); + } + rows.add(t); + } + currentRow.clear(); + } + + /** + * Creates and returns a <code>Reader</code> object which contains the current set + * of header (optional) and data rows. The data will be formatted according to the + * current set of configurable attributes (e.g. delimiter settings). + * @return Reader which contains the formatted header and data rows. + */ + public Reader buildReader() { + StringBuilder sb = new StringBuilder(); + if (!suppressHeader) { + addHeader(sb); + } + flushCurrentRow(); + addRows(sb); + return new StringReader(sb.toString()); + } + + /** + * Helper method for adding (optional) header data to given <code>StringBuilder</code>. + * @param sb <code>StringBuilder</code> to append header row. + */ + private void addHeader(StringBuilder sb) { + sb.append(join(header)); + sb.append(getRowDelimiter()); + } + + /** + * Helper methos for adding data rows to the given <code>StringBuilder</code>. + * @param sb <code>StringBuilder</code> to append data rows. + */ + private void addRows(StringBuilder sb) { + for (List<String> row: rows) { + sb.append(join(row)); + sb.append(getRowDelimiter()); + } + } + + /** + * Helper method that optionally adds the configured quote delimiter to the given + * <code>String</code>. + * @param h The <code>String</code> to optionally quote. + * @return The optionally quoted <code>String</code> + */ + private String quote(String h) { + String quoted = h; + if (!suppressQoutes) { + quoted = getQuoteDelimter() + h + getQuoteDelimter(); + } + return quoted; + } + + /** + * Helper method that joins the given collection of <code>String</code> using the + * configured column delimiter. + * @param s the collection of strings to join + * @return String containing the collection's elements separated by the column delimiter. + */ + public String join(Collection<String> s) { + if (s == null || s.isEmpty()) return ""; + Iterator<String> iter = s.iterator(); + StringBuilder builder = new StringBuilder(quote(iter.next())); + while( iter.hasNext() ) + { + builder.append(getColumnDelimiter()).append(quote(iter.next())); + } + return builder.toString(); + } + + /** + * Test driver method for this class. [Not exhaustive.] + * @param args + * @throws Exception + */ + public static void main(String[] args) throws Exception { + CSVReaderBuilder cb = new CSVReaderBuilder(); + cb.header("A").header("B").header("C"); + cb.newRow().column("a").column("b").column("c"); + cb.newRow().column("x").column("y").column("z"); + cb.newRow().column(""); + cb.newRow().column("#"); + Reader r = cb.buildReader(); + System.out.println("Default listing..."); + listReader(r); + cb.setSuppressQoutes(true); + r = cb.buildReader(); + System.out.println("No quotes listing..."); + listReader(r); + cb.setSuppressHeader(true); + r = cb.buildReader(); + System.out.println("No quotes and no header listing..."); + listReader(r); + cb.setColumnDelimiter("\t"); + r = cb.buildReader(); + System.out.println("No quotes and no header and tab delimited listing..."); + listReader(r); + } + + /** + * Helper method for displaying content of the given <code>Reader</code> + * to <code>System.out</code>. + * @param r the <code>Reader</code> to read from. + * @throws IOException if there's a problem obtaining data from the <code>Reader</code>. + */ + private static void listReader(Reader r) throws IOException { + BufferedReader br = new BufferedReader(r); + String s = null; + while((s = br.readLine()) != null) { + System.out.println(s); + } + } +} + Modified: branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/TestAll.java =================================================================== --- branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/TestAll.java 2010-09-24 17:46:05 UTC (rev 3627) +++ branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/TestAll.java 2010-09-24 18:26:07 UTC (rev 3628) @@ -73,8 +73,6 @@ // Note: class is not debugged and is marked as deprecated, test is commented out. // suite.addTestSuite( TestHybridTimestampFactory.class ); - suite.addTestSuite(TestByteBufferBitVector.class); - suite.addTestSuite( TestCSVReader.class ); suite.addTestSuite( TestBootStateUtil.class ); return suite; Deleted: branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/TestByteBufferBitVector.java =================================================================== --- branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/TestByteBufferBitVector.java 2010-09-24 17:46:05 UTC (rev 3627) +++ branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/TestByteBufferBitVector.java 2010-09-24 18:26:07 UTC (rev 3628) @@ -1,197 +0,0 @@ -/* - -Copyright (C) SYSTAP, LLC 2006-2008. All rights reserved. - -Contact: - SYSTAP, LLC - 4501 Tower Road - Greensboro, NC 27410 - lic...@bi... - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ -/* - * Created on Aug 6, 2009 - */ - -package com.bigdata.util; - -import it.unimi.dsi.bits.BitVector; - -import java.nio.ByteBuffer; - -import com.bigdata.util.ByteBufferBitVector; - -import junit.framework.TestCase2; - -/** - * Test suite for {@link ByteBufferBitVector}. - * - * @author <a href="mailto:tho...@us...">Bryan Thompson</a> - * @version $Id$ - */ -public class TestByteBufferBitVector extends TestCase2 { - - /** - * - */ - public TestByteBufferBitVector() { - } - - /** - * @param name - */ - public TestByteBufferBitVector(String name) { - super(name); - } - - /** Correct rejection test for ctor1. */ - public void test_ctor1_correct_rejection() { - - try { - new ByteBufferBitVector(null); - fail("Expecting: " + IllegalArgumentException.class); - } catch (IllegalArgumentException ex) { - if (log.isInfoEnabled()) - log.info("ignoring expected exception: " + ex); - } - - } - - public void test_ctor1() { - - final byte[] d = new byte[1]; - final ByteBuffer b = ByteBuffer.wrap(d); - final BitVector v = new ByteBufferBitVector(b); - - assertEquals("length", 8L, v.length()); - - // verify range check. - try { - v.getBoolean(-1); - fail("Expecting: " + IndexOutOfBoundsException.class); - } catch (IndexOutOfBoundsException ex) { - if (log.isInfoEnabled()) - log.info("Ignoring expected exception: " + ex); - } - - // verify range check. - try { - v.getBoolean(8); - fail("Expecting: " + IndexOutOfBoundsException.class); - } catch (IndexOutOfBoundsException ex) { - if (log.isInfoEnabled()) - log.info("Ignoring expected exception: " + ex); - } - - for (long i = 0; i < 8L; i++) - assertEquals(false, v.getBoolean(i)); - - // set bit zero. - d[0] |= (1 << 0); - if (log.isInfoEnabled()) - log.info(v.toString()); - assertEquals(true, v.getBoolean(0)); - - // clear bit zero. - d[0] &= ~(1 << 0); - if (log.isInfoEnabled()) - log.info(v.toString()); - assertEquals(false, v.getBoolean(0)); - - } - - /** - * Correct rejection and assumptions for ctor accepting offset and length - * options. - * - * @todo this tests with an even byte offset. Try w/ only a few bits offset. - */ - public void test_ctor2() { - - final byte[] d = new byte[3]; - final ByteBuffer b = ByteBuffer.wrap(d); - final BitVector v = new ByteBufferBitVector(b, 8, 8); - - assertEquals("length", 8L, v.length()); - - // verify range check. - try { - v.getBoolean(-1); - fail("Expecting: " + IndexOutOfBoundsException.class); - } catch (IndexOutOfBoundsException ex) { - if (log.isInfoEnabled()) - log.info("Ignoring expected exception: " + ex); - } - - // verify range check. - try { - v.getBoolean(8); - fail("Expecting: " + IndexOutOfBoundsException.class); - } catch (IndexOutOfBoundsException ex) { - if (log.isInfoEnabled()) - log.info("Ignoring expected exception: " + ex); - } - - for (long i = 0; i < 8L; i++) - assertEquals(false, v.getBoolean(i)); - - // set bit zero. - d[1] |= (1 << 0); - if (log.isInfoEnabled()) - log.info(v.toString()); - assertEquals(true, v.getBoolean(0)); - - // clear bit zero. - d[1] &= ~(1 << 0); - if (log.isInfoEnabled()) - log.info(v.toString()); - assertEquals(false, v.getBoolean(0)); - - } - - /** - * Verify set/clear of each bit in the first byte. - */ - public void test_getBoolean() { - - final byte[] d = new byte[1]; - final ByteBuffer b = ByteBuffer.wrap(d); - final BitVector v = new ByteBufferBitVector(b); - - // verify all bits are zero. - for (long i = 0; i < 8L; i++) - assertEquals(false, v.getBoolean(i)); - - // set/clear each bit in the first byte in turn. - for (int i = 0; i < 8; i++) { - - // set bit - d[0] |= (1 << i); - if (log.isInfoEnabled()) - log.info(v.toString() + " : i=" + i + ", (1<<" + i + ")=" - + (i << i)); - assertEquals(true, v.getBoolean(i)); - - // clear bit - d[0] &= ~(1 << i); - if (log.isInfoEnabled()) - log.info(v.toString()); - assertEquals(false, v.getBoolean(i)); - - } - - } - -} Modified: branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/TestCSVReader.java =================================================================== --- branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/TestCSVReader.java 2010-09-24 17:46:05 UTC (rev 3627) +++ branches/bbb_cleanup/bigdata-core/src/test/java/com/bigdata/util/TestCSVReader.java 2010-09-24 18:26:07 UTC (rev 3628) @@ -29,10 +29,20 @@ import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.io.StringReader; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; import java.util.Iterator; +import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.TreeMap; import junit.framework.TestCase2; @@ -64,23 +74,41 @@ super(name); } - public void test_ctor_correctRejection() throws IOException { + public void test_ctor1_correctRejection() + throws IOException, SecurityException, NoSuchMethodException, + InstantiationException, IllegalAccessException, InvocationTargetException + { + Object[][] cmdLines = { + new Object[] {null, null}, + new Object[] {null, "UTF-8"}, + new Object[] {new ByteArrayInputStream(new byte[]{}), null}, + }; - try { - new CSVReader(null,"UTF-8"); - fail("Expecting: "+IllegalArgumentException.class); - } catch(IllegalArgumentException ex) { - log.info("Ignoring expected exception: "+ex); + Constructor<CSVReader> cons = + CSVReader.class.getConstructor(InputStream.class, String.class); + for (Object[] cmdLine: cmdLines) { + try { + cons.newInstance(cmdLine); + fail("Expecting: " + IllegalArgumentException.class.toString()); + } catch(InvocationTargetException ex) { + if (!(ex.getCause() instanceof IllegalArgumentException)) { + fail("Expecting: " + IllegalArgumentException.class.toString()); + } + //ignore -- expected + } } + } + + public void test_ctor2_correctRejection() throws IOException { try { - new CSVReader(new ByteArrayInputStream(new byte[]{}),null); - fail("Expecting: "+IllegalArgumentException.class); + Reader r = null; + new CSVReader(r); + fail("Expecting: " + IllegalArgumentException.class.toString()); } catch(IllegalArgumentException ex) { - log.info("Ignoring expected exception: "+ex); + //ignore -- expected } - - } + } /** * Test reads from a tab-delimited file <code>test.csv</code> with headers @@ -113,7 +141,7 @@ assertEquals(1,r.lineNo()); - assertEquals(headers, r.headers); + assertEquals(headers, r.getHeaders()); /* * 1st row of data. @@ -206,7 +234,359 @@ assertFalse(r.hasNext()); } + + private static Header[] convertStringToHeader(String[] sa) { + Header[] h = new Header[sa.length]; + int i=0; + for (String s: sa) { + h[i++] = new Header(s); + } + return h; + } + private static String[] stringHeaders = { "Header1", "Header2", "Header3" }; + private static String[] defaultStringHeaders = { "1", "2", "3" }; + + private static Header[] headers = convertStringToHeader(stringHeaders); + private static Header[] defaultHeaders = convertStringToHeader(defaultStringHeaders); + + private static Object[][] rows = { + { "Column11", "Column12", "Column13" }, + { "Column21", "Column22", "Column23" }, + { "Column31", "Column32", "Column33" }, + { "Column with spaces", "and more spaces", "and embedded \"quotes\"" }, + }; + + private static CSVReaderBuilder getDefaultTestCSVReaderBuilder() { + CSVReaderBuilder b = new CSVReaderBuilder(); + for (String header: stringHeaders) { + b.header(header); + } + for (Object[] row: rows) { + b.newRow(); + for (Object col: row) { + b.column(col.toString()); + } + } + return b; + } + + public void test_default_csv_reader_with_defaults() throws IOException { + CSVReaderBuilder cb = getDefaultTestCSVReaderBuilder(); + verify_data_and_header(new CSVReader(cb.buildReader())); + } + public void test_default_csv_reader_with_tabs() throws IOException { + CSVReaderBuilder cb = getDefaultTestCSVReaderBuilder(); + cb.setColumnDelimiter("\t"); + verify_data_and_header(new CSVReader(cb.buildReader())); + } + public void test_default_csv_reader_without_quotes() throws IOException { + CSVReaderBuilder cb = getDefaultTestCSVReaderBuilder(); + cb.setSuppressQoutes(true); + verify_data_and_header(new CSVReader(cb.buildReader())); + } + public void test_default_csv_reader_no_headers() throws IOException { + CSVReaderBuilder cb = getDefaultTestCSVReaderBuilder(); + cb.setSuppressHeader(true); + verify_data(new CSVReader(cb.buildReader()),defaultHeaders); + } + + private void verify_data_and_header(CSVReader cr) throws IOException { + // Read and verify header row + assertTrue(cr.hasNext()); + cr.readHeaders(); + assertEquals(cr.getHeaders(), headers); + verify_data(cr, headers); + } + + private void verify_data(CSVReader cr, Header[] headers) throws IOException { + //Read and verify data rows + for (int i=0; i < rows.length; i++) { + assertTrue(cr.hasNext()); + assertSameValues( newMap(headers, rows[i]), cr.next() ); + } + assertFalse(cr.hasNext()); + } + + public void test_header_cons_with_bad_args() { + try { + new Header(null); + fail("Constructed Header with null arg."); + } catch (IllegalArgumentException e){ + //ignore -- expected + } + try { + new Header(""); + fail("Constructed Header with empty arg."); + } catch (IllegalArgumentException e){ + //ignore -- expected + } + } + + public void test_header_cons_with_good_arg() { + String name = "abc"; + Header h = new Header(name); + assertEquals(h.getName(), name); + } + + public void test_header_equals() { + String name = "abc"; + Header h = new Header(name); + Header h_dup = new Header(name); + Header h_dup2 = new Header(name); + Header h_diff = new Header(name + "diff"); + + // Test reflexive property + assertTrue(h.equals(h)); + + // Test symmetric property + assertTrue(h.equals(h_dup) && h_dup.equals(h)); + + //Test transitive property + assertTrue(h.equals(h_dup) && h_dup.equals(h_dup2) && h.equals(h_dup2)); + + // consistency property already tested + + // Test negative cases + assertFalse(h.equals(null)); + + assertFalse(h.equals(name)); + + assertFalse(h.equals(h_diff)); + } + + public void test_header_hashcode() { + String name = "abc"; + Header h = new Header(name); + Header h_dup = new Header(name); + + assertTrue(h.hashCode()==h_dup.hashCode()); + } + + public void test_header_toString() { + String name = "abc"; + Header h = new Header(name); + Header h_dup = new Header(name); + + assertTrue(h.toString().equals(name)); + } + + public void test_setTailDelayMillis_bad_arg() throws IOException { + CSVReaderBuilder cb = getDefaultTestCSVReaderBuilder(); + CSVReader r = new CSVReader(cb.buildReader()); + try { + r.setTailDelayMillis(-1L); + fail("Created CSVReader with negative delay."); + } catch (IllegalArgumentException e) { + //ignore -- expected + } + } + + public void test_setTailDelayMillis_good_arg() throws IOException { + CSVReaderBuilder cb = getDefaultTestCSVReaderBuilder(); + CSVReader r = new CSVReader(cb.buildReader()); + long delay = 1L; + long oldDelay = 0L; + long tmpDelay = 0L; + tmpDelay = r.setTailDelayMillis(delay); + assertTrue(r.getTailDelayMillis()==delay); + assertTrue(tmpDelay==oldDelay); + } + + private static class DelayedReader extends StringReader { + + private boolean isReady = false; + private int maxDelayCount = 3; + private int delayCount = 1; + + public DelayedReader(String s) { + super(s); + } + + @Override + public boolean ready() { + if (delayCount++ > maxDelayCount) isReady = true; + return isReady; + } + + } + + public void test_delay_with_reader() throws IOException { + CSVReaderBuilder crb = new CSVReaderBuilder(); + StringReader s = + new DelayedReader( + crb.join(Arrays.asList(stringHeaders))); + CSVReader r = new CSVReader(s); + r.setTailDelayMillis(1000); // 1 sec + assertTrue(r.hasNext()); + r.readHeaders(); + Header[] actualHeaders = r.getHeaders(); + Header[] expectedHeaders = headers; + assertEquals(expectedHeaders, actualHeaders); + } + + public void test_delay_with_reader_with_comments_and_empty_lines() + throws IOException + { + CSVReaderBuilder crb = new CSVReaderBuilder(); + crb.header("H1").header("H2").header("H3"); + crb.newRow().column("c1").column("c2").column("c3"); + crb.newRow().column("# Comment line"); + crb.newRow().column(""); //Blank line + crb.newRow().column("d1").column("d2").column("d3"); + crb.setSuppressQoutes(true); // Otherwise # isn't first char + CSVReader r = new CSVReader(crb.buildReader()); + assertTrue(r.hasNext()); + r.readHeaders(); + Header[] actualHeaders = r.getHeaders(); + Header[] expectedHeaders = new Header[] { + new Header("H1"), + new Header("H2"), + new Header("H3"), + }; + assertEquals(expectedHeaders, actualHeaders); + //Check that two rows of data gets returned + Map<String, Object> actualRow = r.next(); + Map<String, Object> expectedRow = + newMap(expectedHeaders, + new Object[] { "c1", "c2", "c3"} ); + assertSameValues(expectedRow, actualRow); + actualRow = r.next(); + expectedRow = + newMap(expectedHeaders, + new Object[] { "d1", "d2", "d3"} ); + assertSameValues(expectedRow, actualRow); + assertFalse(r.hasNext()); + try { + r.next(); + fail("Successfully called next() on an empty reader."); + } catch (NoSuchElementException e) { + //ignore -- expected + } + } + + public void test_get_headers() + throws IOException + { + CSVReader r = new CSVReader(new StringReader("")); + assertNull(r.getHeaders()); + } + + public void test_get_headers2() + throws IOException + { + CSVReaderBuilder crb = new CSVReaderBuilder(); + CSVReader r = + new CSVReader( + new StringReader( + crb.join(Arrays.asList(stringHeaders)))); + r.readHeaders(); + Header[] actual = r.getHeaders(); + assertEquals(headers, actual); + } + + public void test_set_headers_null() + throws IOException + { + CSVReaderBuilder crb = new CSVReaderBuilder(); + CSVReader r = + new CSVReader( + new StringReader( + crb.join(Arrays.asList(stringHeaders)))); + try { + r.setHeaders(null); + fail("Was able to set null headers."); + } catch (IllegalArgumentException e) { + //ignore -- expected + } + } + + public void test_set_headers() + throws IOException + { + CSVReaderBuilder crb = new CSVReaderBuilder(); + CSVReader r = + new CSVReader( + new StringReader( + crb.join(Arrays.asList(stringHeaders)))); + r.readHeaders(); + Header[] actual = r.getHeaders(); + assertEquals(headers, actual); + r.setHeaders(defaultHeaders); + actual = r.getHeaders(); + assertEquals(defaultHeaders, actual); + + } + + public void test_set_header_out_of_bounds() + throws IOException + { + CSVReaderBuilder crb = new CSVReaderBuilder(); + CSVReader r = + new CSVReader( + new StringReader( + crb.join(Arrays.asList(stringHeaders)))); + r.readHeaders(); + Header[] actual = r.getHeaders(); + assertEquals(headers, actual); + try { + r.setHeader(stringHeaders.length, new Header("out-of-bounds")); + fail("Able to set an out-of-bounds header element."); + } catch (IndexOutOfBoundsException e) { + //ignore -- expected + } + } + public void test_set_header_null() + throws IOException + { + CSVReaderBuilder crb = new CSVReaderBuilder(); + CSVReader r = + new CSVReader( + new StringReader( + crb.join(Arrays.asList(stringHeaders)))); + r.readHeaders(); + Header[] actual = r.getHeaders(); + assertEquals(headers, actual); + try { + r.setHeader(stringHeaders.length-1, null); + fail("Able to set a null header element."); + } catch (IllegalArgumentException e) { + //ignore -- expected + } + } + + public void test_set_header_valid() + throws IOException + { + CSVReaderBuilder crb = new CSVReaderBuilder(); + CSVReader r = + new CSVReader( + new StringReader( + crb.join(Arrays.asList(stringHeaders)))); + r.readHeaders(); + Header[] actual = r.getHeaders(); + assertEquals(headers, actual); + Header[] headersClone = headers.clone(); + int last = headersClone.length-1; + headersClone[last] = new Header("replacement"); + r.setHeader(last, headersClone[last]); + actual = r.getHeaders(); + assertEquals(headersClone, actual); + } + + public void test_remove() + throws IOException + { + CSVReader r = + new CSVReader(new StringReader("bogus")); + try { + r.remove(); + fail("Successfully called unsupported operation."); + } catch (UnsupportedOperationException e) { + // ignore -- expected + } + } + protected void assertEquals(Header[] expected, Header[] actual) { assertEquals(expected.length,actual.length); @@ -215,7 +595,7 @@ if(!expected[i].equals( actual[i])) { - fail("headers["+i+"], expected ["+expected[i]+"]u not ["+actual[i]+"]" ); + fail("headers["+i+"], expected ["+expected[i]+"] not ["+actual[i]+"]" ); } This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |