[Batchserver-cvs] batchserver/src/org/jmonks/batch/io/flat DelimitedFlatFileFieldSpec.java, NONE,
Brought to you by:
suresh_pragada
Update of /cvsroot/batchserver/batchserver/src/org/jmonks/batch/io/flat In directory sc8-pr-cvs3.sourceforge.net:/tmp/cvs-serv20420 Added Files: DelimitedFlatFileFieldSpec.java DelimitedFlatFileFileSpec.java DelimitedFlatFileReader.java DelimitedFlatFileRecordSpec.java DelimitedFlatFileWriter.java FixedWidthFlatFileFieldSpec.java FixedWidthFlatFileFileSpec.java FixedWidthFlatFileReader.java FixedWidthFlatFileRecordSpec.java FixedWidthFlatFileWriter.java FlatFileReader.java FlatFileWriter.java package.html sample-delimited-file-spec.xml sample-delimited-file.dat sample-fixed-width-file-spec.xml sample-fixed-width-file.dat Log Message: no message --- NEW FILE: DelimitedFlatFileRecordSpec.java --- /* * DelimitedFlatFileRecordSpec.java * * Created on June 10, 2006, 10:55 AM * * To change this template, choose Tools | Options and locate the template under * the Source Creation and Management node. Right-click the template and choose * Open. You can then make changes to the template in the Source Editor. */ package org.jmonks.batch.io.flat; import java.util.Iterator; import org.apache.log4j.Logger; import org.jmonks.batch.io.FieldSpec; import org.jmonks.batch.io.FileSpecException; import org.jmonks.batch.io.RecordSpec; import org.jmonks.batch.io.RecordType; import org.w3c.dom.Element; import org.w3c.dom.NodeList; /** * <p> * DelimitedFlatFileRecordSpec represents record-spec element in the file spec * belongs to the delimited flat file type. Along with the <code>record-type</code> attribute * it looks for the two additional attributes <code>delimiter</code> and <code>field-count</code>. * Attribute <code>delimiter</code> tells the value that divides or delimits the fields * and attribute <code>field-count</code> tells the number of fields exists in the record. * There should be only one record spec is allowed in this file spec. * Here is a sample spec snippet... * </p> * <p> * <pre> * <file-spec file-type="delimited-flat"> * <record-spec record-type="detail" delimiter="|" field-count="4"> * <!-- field specs will follow here --> * </record-spec> * </file-spec> * </pre> * </p> * * @author Suresh Pragada * @version 1.0 * @since 1.0 */ public class DelimitedFlatFileRecordSpec extends RecordSpec { /** * Represents the delimiter for the record. */ private String delimiter=null; /** * Represents the number of fields available in the record. */ private int fieldCount; /** * Constant defines the delimiter attribute name which the value is "delimiter" */ public static final String DELIMITER_ATTRIB_NAME = "delimiter"; /** * Constant defines the field count attribute name which the value is "field-count" */ public static final String FIELD_COUNT_ATTRIB_NAME = "field-count"; private static Logger logger=Logger.getLogger(DelimitedFlatFileRecordSpec.class); /** * Constructs the record spec by accepting the record type. */ protected DelimitedFlatFileRecordSpec(RecordType recordType) { super(recordType); } /** * Returns the delimiter which divides the fields in the record. */ public String getDelimiter() { return this.delimiter; } /** * Returns the number of fields exists in this record. */ public int getFieldCount() { return this.fieldCount; } /** * Factory method to create the delimited flat record spec from the given * DOM Element representing the record-spec element in the file spec. * * @param recordSpecElement DOM Element representing the record spec. * * @return Returns the delimited flat file record spec. * * @throws org.jmonks.batch.io.FileSpecException If record-spec * doesnt have the delimiter or record-type attribute or index value is repeated on more than one field spec. */ public static DelimitedFlatFileRecordSpec createDelimitedFlatFileRecordSpec(final Element recordSpecElement) { logger.trace("Entering createDelimitedFlatFileRecordSpec"); DelimitedFlatFileRecordSpec recordSpec=null; String configuredRecordType=recordSpecElement.getAttribute(RecordSpec.RECORD_TYPE_ATTRIB_NAME); logger.debug("record specs record-type value = " + configuredRecordType); if(configuredRecordType!=null && !(configuredRecordType.trim().equals(""))) { RecordType recordType=RecordType.toRecordType(configuredRecordType); recordSpec=new DelimitedFlatFileRecordSpec(recordType); } else throw new FileSpecException("Record Spec in Delimited File Spec should have record-type attribute."); String delimiter=recordSpecElement.getAttribute(DelimitedFlatFileRecordSpec.DELIMITER_ATTRIB_NAME); logger.debug("record specs delimiter value = " + delimiter); if(delimiter!=null && !"".equals(delimiter)) recordSpec.delimiter=delimiter; else throw new FileSpecException("Record Spec in Delimited File Spec should have delimiter attribute."); String fieldCount=recordSpecElement.getAttribute(DelimitedFlatFileRecordSpec.FIELD_COUNT_ATTRIB_NAME); logger.debug("record specs field count value = " + delimiter); if(fieldCount!=null && !"".equals(fieldCount.trim())) recordSpec.fieldCount=Integer.parseInt(fieldCount); else throw new FileSpecException("Record Spec in Delimited File Spec should have field-count attribute."); NodeList fieldSpecNodeList=recordSpecElement.getElementsByTagName(FieldSpec.FIELD_SPEC_TAG_NAME); for(int i=0;i<fieldSpecNodeList.getLength();i++) { DelimitedFlatFileFieldSpec fieldSpec=DelimitedFlatFileFieldSpec.createDelimitedFlatFileFieldSpec((Element)fieldSpecNodeList.item(i)); for(Iterator iterator=recordSpec.getFieldSpecs().iterator();iterator.hasNext();) { DelimitedFlatFileFieldSpec existingFieldSpec=(DelimitedFlatFileFieldSpec)iterator.next(); if(existingFieldSpec.getIndex()==fieldSpec.getIndex()) { throw new FileSpecException("Found the index value " + fieldSpec.getIndex() + " on multiple fields."); } } recordSpec.addFieldSpec(fieldSpec); } return recordSpec; } /** * @see java.lang.Object#toString() */ public String toString() { StringBuffer stringValue=new StringBuffer("{DelimitedFlatFileRecordSpec "); stringValue.append("[recordType = " + super.recordType.toString() + "]"); stringValue.append("[delimiter = " + this.delimiter + "]"); stringValue.append("[fieldCount = " + this.fieldCount + "]"); stringValue.append("[fieldSpecList = "); for(Iterator iterator=fieldSpecList.iterator();iterator.hasNext();) stringValue.append(((DelimitedFlatFileFieldSpec)iterator.next()).toString()); stringValue.append("]}"); return stringValue.toString(); } } --- NEW FILE: FixedWidthFlatFileWriter.java --- /* * FixedWidthFlatFileWriter.java * * Created on June 6, 2006, 8:58 AM * * To change this template, choose Tools | Options and locate the template under * the Source Creation and Management node. Right-click the template and choose * Open. You can then make changes to the template in the Source Editor. */ package org.jmonks.batch.io.flat; import java.io.IOException; import java.io.OutputStream; import java.io.Writer; import java.util.Arrays; import java.util.Iterator; import java.util.List; import org.apache.log4j.Logger; import org.jmonks.batch.io.FileSpec; import org.jmonks.batch.io.WriterRecord; import org.jmonks.batch.io.flat.FlatFileWriter.FlatFileWriterRecord; /** * <p> * FixedWidthFlatFileWriter writes the file according to the given file spec * with the data submitted in the form of WriterRecord's. This provides the methods * to create the required writer records and write the writer record into the file writer. * Once finished writing of all the records file writer should be closed by calling * the "close" method. * </p> * * @author Suresh Pragada * @version 1.0 * @since 1.0 */ public class FixedWidthFlatFileWriter extends FlatFileWriter { private static Logger logger=Logger.getLogger(FixedWidthFlatFileWriter.class); /** * Constructs and initializes the writer with the given values. * * @param outputStream Output stream to write the records. * @param fileSpec File spec to be used to generate the file. */ public FixedWidthFlatFileWriter(OutputStream outputStream,FileSpec fileSpec) { super(outputStream,fileSpec); } /** * Constructs and initializes the writer with the given values. * * @param writer Writer to write the records. * @param fileSpec File spec to be used to generate the file. */ public FixedWidthFlatFileWriter(Writer writer,FileSpec fileSpec) { super(writer,fileSpec); } /** * Writes the given record into the file/writer. * * @param writerRecord Record consist of field names and values. * * @throws IllegalStateException If writer is closed and trying to write the recrod. * @throws IllegalArgumentException If writer record is null. * @throws org.jmonks.batch.io.FileParseException Problems while trying to write the record. */ protected String generateRecord(WriterRecord writerRecord) throws IOException { logger.trace("Entering generateRecord"); FlatFileWriterRecord record=(FlatFileWriterRecord)writerRecord; FixedWidthFlatFileRecordSpec recordSpec=(FixedWidthFlatFileRecordSpec)this.fileSpec.getRecordSpec(record.getRecordType()); char[] recordBuffer=new char[recordSpec.getRecordLength()]; Arrays.fill(recordBuffer,' '); System.arraycopy(recordSpec.startsWith.toCharArray(), 0, recordBuffer, 0, recordSpec.startsWith.length()); List fieldSpecList=recordSpec.getFieldSpecs(); for(Iterator iterator=fieldSpecList.iterator();iterator.hasNext();) { FixedWidthFlatFileFieldSpec fieldSpec=(FixedWidthFlatFileFieldSpec)iterator.next(); String fieldValue=(String)record.readField(fieldSpec.getFieldName()); if(fieldValue!=null) { char[] fieldValueChars=fieldValue.toCharArray(); System.arraycopy(fieldValueChars, 0, recordBuffer, fieldSpec.getStartPosition()-1, ((fieldValueChars.length>fieldSpec.getFieldWidth())?fieldSpec.getFieldWidth():fieldValueChars.length)); } } logger.trace("Exiting generateRecord"); return new String(recordBuffer); } } --- NEW FILE: DelimitedFlatFileWriter.java --- /* * DelimitedFlatFileWriter.java * * Created on June 10, 2006, 12:38 PM * * To change this template, choose Tools | Options and locate the template under * the Source Creation and Management node. Right-click the template and choose * Open. You can then make changes to the template in the Source Editor. */ package org.jmonks.batch.io.flat; import java.io.OutputStream; import java.io.Writer; import java.util.Iterator; import org.apache.log4j.Logger; import org.jmonks.batch.io.FileSpec; import org.jmonks.batch.io.WriterRecord; import org.jmonks.batch.io.flat.FlatFileWriter.FlatFileWriterRecord; /** * <p> * DelimitedFlatFileWriter writes the file according to the given file spec * with the data submitted in the form of WriterRecord's. This provides the methods * to create the required writer records and write the writer record into the file writer. * Once finished writing of all the records file writer should be closed by calling * the "close" method. * </p> * * @author Suresh Pragada * @version 1.0 * @since 1.0 */ public class DelimitedFlatFileWriter extends FlatFileWriter { private static Logger logger=Logger.getLogger(DelimitedFlatFileWriter.class); /** * Holds the single record spec exists in Delimited file spec. */ private DelimitedFlatFileRecordSpec recordSpec=null; /** * Constructs the Delimited flat file writer from the given output stream * and file spec. * * @param outputStream Output stream representing the file to generate. * @param fileSpec File spec of the delimited flat to be generated. */ public DelimitedFlatFileWriter(OutputStream outputStream,FileSpec fileSpec) { super(outputStream,fileSpec); recordSpec=(DelimitedFlatFileRecordSpec)fileSpec.getRecordSpecs().iterator().next(); } /** * Constructs the Delimited flat file writer from the given output stream * and file spec. * * @param writer Writer representing the file to generate. * @param fileSpec File spec of the delimited flat to be generated. */ public DelimitedFlatFileWriter(Writer writer,FileSpec fileSpec) { super(writer,fileSpec); recordSpec=(DelimitedFlatFileRecordSpec)fileSpec.getRecordSpecs().iterator().next(); } /** * Generates the string represenatation of the record from the writer record given by client. * * @param writerRecord Writer record consists of field names and values. * * @return Returns the string representation of the record. */ protected String generateRecord(WriterRecord writerRecord) throws java.io.IOException { FlatFileWriterRecord flatFileWriterRecord=(FlatFileWriterRecord)writerRecord; String[] fieldValuesArray=new String[recordSpec.getFieldCount()]; for(Iterator iterator=recordSpec.getFieldSpecs().iterator();iterator.hasNext();) { DelimitedFlatFileFieldSpec fieldSpec=(DelimitedFlatFileFieldSpec)iterator.next(); String fieldValue=(String)flatFileWriterRecord.readField(fieldSpec.getFieldName()); fieldValuesArray[fieldSpec.getIndex()-1]= fieldValue!=null?fieldValue:""; } StringBuffer recordBuffer=new StringBuffer(); for(int i=0;i<fieldValuesArray.length;i++) { if(i==fieldValuesArray.length-1) recordBuffer.append(fieldValuesArray[i]); else { recordBuffer.append(fieldValuesArray[i]); recordBuffer.append(recordSpec.getDelimiter()); } } return recordBuffer.toString(); } } --- NEW FILE: sample-delimited-file-spec.xml --- <?xml version="1.0" encoding="UTF-8"?> <file-spec file-type="delimited-flat"> <record-spec record-type="detail" delimiter="|" field-count="4"> <field-spec field-name="field1" index="1"/> <field-spec field-name="field2" index="2"/> <field-spec field-name="field3" index="3"/> <field-spec field-name="field4" index="4"/> </record-spec> </file-spec> --- NEW FILE: DelimitedFlatFileFileSpec.java --- /* * DelimitedFlatFileFileSpec.java * * Created on June 10, 2006, 10:47 AM * * To change this template, choose Tools | Options and locate the template under * the Source Creation and Management node. Right-click the template and choose * Open. You can then make changes to the template in the Source Editor. */ package org.jmonks.batch.io.flat; import java.util.Iterator; import org.apache.log4j.Logger; import org.jmonks.batch.io.FileSpec; import org.jmonks.batch.io.FileSpecException; import org.jmonks.batch.io.FileType; import org.jmonks.batch.io.RecordSpec; import org.w3c.dom.Element; import org.w3c.dom.NodeList; /** * <p> * DelimitedFlatFileFileSpec represents the file spec defines the flat file * where all the fields in the records delimited with a particular character or value. * This file spec doesn't require any special attributes other than <code>file-type</code> attribute, * which should be "delimited-flat". * Here is a sample file spec... * </p> * <p> * <pre> * <file-spec file-type="delimited-flat"> * <!-- record specs will follow here --> * </file-spec> * </pre> * </p> * * @author Suresh Pragada * @version 1.0 * @since 1.0 */ public class DelimitedFlatFileFileSpec extends FileSpec { private static Logger logger=Logger.getLogger(DelimitedFlatFileFileSpec.class); /** * Constructs the DelimitedFlatFileFileSpec. */ protected DelimitedFlatFileFileSpec() { super(FileType.DELIMITED_FLAT_FILE); } /** * Factory method create the delimited flat file spec object from the given * DOM Element representing the file-spec element. * * @param fileSpecElement DOM Element representing the file-spec element. * * @return Returns the instance of DelimitedFlatFileFileSpec. * * @throws org.jmonks.batch.io.FileSpecException If more than one record spec is configured. */ public static FileSpec createDelimitedFlatFileFileSpec(final Element fileSpecElement) { logger.trace("Entering createDelimitedFlatFileFileSpec"); DelimitedFlatFileFileSpec fileSpec=new DelimitedFlatFileFileSpec(); NodeList recordSpecNodeList=fileSpecElement.getElementsByTagName(RecordSpec.RECORD_SPEC_TAG_NAME); if(recordSpecNodeList.getLength()>1) { logger.fatal("Delimited file spec should not have more than 1 record spec. Number of record specs found = " + recordSpecNodeList.getLength()); throw new FileSpecException("Delimited file spec should have only one record spec."); } else if(recordSpecNodeList.getLength()<1) { logger.fatal("Delimited file spec should have atleast one record spec."); throw new FileSpecException("Delimited file spec should have atleast one record spec."); } else { DelimitedFlatFileRecordSpec recordSpec=DelimitedFlatFileRecordSpec.createDelimitedFlatFileRecordSpec((Element)recordSpecNodeList.item(0)); logger.debug("Adding the record spec = " + recordSpec.toString()); fileSpec.addRecordSpec(recordSpec); return (FileSpec)fileSpec; } } /** * @see java.lang.Object#toString() */ public String toString() { StringBuffer stringValue=new StringBuffer("{DelimitedFlatFileFileSpec "); stringValue.append("[fileType = " + super.fileType.toString() + "]"); stringValue.append("[recordSpecList = "); for(Iterator iterator=recordSpecMap.values().iterator();iterator.hasNext();) stringValue.append(((DelimitedFlatFileRecordSpec)iterator.next()).toString()); stringValue.append("]}"); return stringValue.toString(); } } --- NEW FILE: FixedWidthFlatFileFileSpec.java --- /* * FixedWidthFlatFileFileSpec.java * * Created on June 1, 2006, 2:06 PM * * To change this template, choose Tools | Options and locate the template under * the Source Creation and Management node. Right-click the template and choose * Open. You can then make changes to the template in the Source Editor. */ package org.jmonks.batch.io.flat; import java.util.Iterator; import org.apache.log4j.Logger; import org.w3c.dom.Element; import org.w3c.dom.NodeList; import org.jmonks.batch.io.FileType; import org.jmonks.batch.io.RecordSpec; import org.jmonks.batch.io.FileSpecException; import org.jmonks.batch.io.FileSpec; /** * <p> * FixedWidthFlatFileFileSpec represents the file spec defines the flat file * where each different kind of record starts with a particular value and the * fields in each record will have start and end position. This file-spec doesn't * require any special attributes other than file-type, which should be "fixed-width-flat". * Here is a sample file spec... * </p> * <p> * <pre> * <file-spec file-type="fixed-width-flat"> * <!-- record specs will follow here --> * </file-spec> * </pre> * </p> * * @author Suresh Pragada * @version 1.0 * @since 1.0 */ public class FixedWidthFlatFileFileSpec extends FileSpec { private static Logger logger=Logger.getLogger(FixedWidthFlatFileFileSpec.class); /** * Constructs the FixedWidthFlatFileFileSpec. */ protected FixedWidthFlatFileFileSpec() { super(FileType.FIXED_WIDTH_FLAT_FILE); } /** * Factory method create the fixed width flat file spec object from the given * DOM Element representing the file-spec element. * * @param fileSpecElement DOM Element representing the file-spec element. * * @return Returns the instance of FixedWidthFlatFileFileSpec. * * @throws org.jmonks.batch.io.FileSpecException If two record specs * has the same value for the starts-with and record-type attributes. */ public static FileSpec createFixedWidthFlatFileFileSpec(final Element fileSpecElement) { logger.trace("Entering createFixedWidthFlatFileFileSpec "); FixedWidthFlatFileFileSpec fileSpec=new FixedWidthFlatFileFileSpec(); NodeList recordSpecNodeList=fileSpecElement.getElementsByTagName(RecordSpec.RECORD_SPEC_TAG_NAME); logger.debug("Number of record specs found = " + recordSpecNodeList.getLength()); for(int i=0;i<recordSpecNodeList.getLength();i++) { FixedWidthFlatFileRecordSpec recordSpec=FixedWidthFlatFileRecordSpec.createFixedWidthFlatFileRecordSpec((Element)recordSpecNodeList.item(i)); /** * Check for the duplicat start-with value on the record specs. */ for(Iterator iterator=fileSpec.getRecordSpecs().iterator();iterator.hasNext();) { FixedWidthFlatFileRecordSpec existingRecordSpec=(FixedWidthFlatFileRecordSpec)iterator.next(); if(existingRecordSpec.getStartsWith().equalsIgnoreCase(recordSpec.getStartsWith())) { throw new FileSpecException("Two record specs in the same file spec cannot have same values for starts-with attribute."); } } logger.debug("Adding the record spec = " + recordSpec.toString()); fileSpec.addRecordSpec(recordSpec); } return (FileSpec)fileSpec; } /** * @see java.lang.Object#toString() */ public String toString() { StringBuffer stringValue=new StringBuffer("{FixedWidthFlatFileFileSpec "); stringValue.append("[fileType = " + super.fileType.toString() + "]"); stringValue.append("[recordSpecList = "); for(Iterator iterator=recordSpecMap.values().iterator();iterator.hasNext();) stringValue.append(((FixedWidthFlatFileRecordSpec)iterator.next()).toString()); stringValue.append("]}"); return stringValue.toString(); } } --- NEW FILE: FixedWidthFlatFileReader.java --- /* * FixedWidthFlatFileReader.java * * Created on May 26, 2006, 2:15 PM * * To change this template, choose Tools | Options and locate the template under * the Source Creation and Management node. Right-click the template and choose * Open. You can then make changes to the template in the Source Editor. */ package org.jmonks.batch.io.flat; import java.io.InputStream; import java.io.Reader; import java.util.Collection; import java.util.Iterator; import java.util.List; import org.apache.log4j.Logger; import org.jmonks.batch.io.FileParseException; import org.jmonks.batch.io.FileSpec; import org.jmonks.batch.io.ReaderRecord; import org.jmonks.batch.io.flat.FlatFileReader.FlatFileReaderRecord; /** * <p> * FixedWidthFlatFileReader reads the specified fixed width flat file according to the given file spec * and returns the recrods on the needed basis. Each field value from the record * should be read using readField method by passing the fieldName mentioned in * the file spec. To find out how to read each record from the file and to read * the each field from the record, refer to the FileReader javadoc. * </p> * * @author Suresh Pragada * @version 1.0 * @since 1.0 */ public class FixedWidthFlatFileReader extends FlatFileReader { private static Logger logger=Logger.getLogger(FixedWidthFlatFileReader.class); /** * Constructs and initializes the reader with the given file path and file spec. * * @param fileInputStream Input stream to the file. * @param fileSpec File spec to be used to read the file. */ public FixedWidthFlatFileReader(InputStream fileInputStream,FileSpec fileSpec) { super(fileInputStream,fileSpec); } /** * Constructs and initializes the reader with the given file path and file spec. * * @param reader Reader to the file. * @param fileSpec File spec to be used to read the file. */ public FixedWidthFlatFileReader(Reader reader,FileSpec fileSpec) { super(reader,fileSpec); } /** * Parses the given record string, translates it into the proper ReaderRecord * and returns the reader record. * * @param recordString Record read from the file. * * @throws org.jmonks.batch.io.FileParseException Problems while parsing the * next record. This includes unable to identify the recod with the availble record specs. */ protected ReaderRecord parseRecord(String recordString) { logger.trace("Entering parseRecord"); FlatFileReaderRecord record=null; Collection recordSpecCollection=this.fileSpec.getRecordSpecs(); for(Iterator recordSpecIterator=recordSpecCollection.iterator();recordSpecIterator.hasNext();) { FixedWidthFlatFileRecordSpec recordSpec=(FixedWidthFlatFileRecordSpec)recordSpecIterator.next(); if(recordSpec.isMatch(recordString)) { List fieldSpecList=recordSpec.getFieldSpecs(); record=new FlatFileReaderRecord(recordSpec.getRecordType(),fieldSpecList.size()); for(Iterator fieldSpecIterator=fieldSpecList.iterator();fieldSpecIterator.hasNext();) { FixedWidthFlatFileFieldSpec fieldSpec=(FixedWidthFlatFileFieldSpec)fieldSpecIterator.next(); String fieldValue=null; try { fieldValue=recordString.substring(fieldSpec.getStartPosition()-1,fieldSpec.getEndPosition()); } catch(IndexOutOfBoundsException exception) { exception.printStackTrace(); logger.fatal("Could not retrieve the field " + fieldSpec.getFieldName() + " from the record string " + recordString + " from the positions start = " + fieldSpec.getStartPosition() + " end = " + fieldSpec.getEndPosition()); } record.writeField(fieldSpec.getFieldName(),fieldValue); } break; } if(!recordSpecIterator.hasNext()) throw new FileParseException("Record " + recordString + " cannot be matched with any configured record specs."); } logger.trace("Exiting parseRecord"); return record; } } --- NEW FILE: FixedWidthFlatFileFieldSpec.java --- /* * FixedWidthFlatFileFieldSpec.java * * Created on May 26, 2006, 2:43 PM * * To change this template, choose Tools | Options and locate the template under * the Source Creation and Management node. Right-click the template and choose * Open. You can then make changes to the template in the Source Editor. */ package org.jmonks.batch.io.flat; import org.apache.log4j.Logger; import org.w3c.dom.Element; import org.jmonks.batch.io.FieldSpec; import org.jmonks.batch.io.FileSpecException; /** * <p> * FixedWidthFlatFileFieldSpec represents <code>field-spec</code> element in the record spec * belongs to the fixed width flat file type. Along with the <code>field-name</code> attribute * it looks for the <code>start-pos</code> and <code>end-pos</code> attributes in the field-spec element to retrieve * the field from the record. Field name on <code>field-spec</code> should be unique across the record-spec. * Position value starts with "1" in the record and the start-pos and end-pos values are inclusive. * Here is a sample spec snippet... * </p> * <p> * <pre> * <file-spec file-type="fixed-width-flat"> * <record-spec record-type="detail" starts-with="5"> * <field-spec field-name="consumer-id" start-pos="2" end-pos="11"/> * <field-spec field-name="consumer-name" start-pos="12" end-pos="31"/> * </record-spec> * </file-spec> * </pre> * </p> * * @author Suresh Pragada * @version 1.0 * @since 1.0 */ public class FixedWidthFlatFileFieldSpec extends FieldSpec { /** * Holds the start position of the field. */ private int startPosition=0; /** * Holds the end position of the field. */ private int endPosition=0; /** * Constant defines the start position attribute name. */ public static final String START_POSITION_ATTRIB_NAME = "start-pos"; /** * Constant defines the end position attribute name. */ public static final String END_POSITION_ATTRIB_NAME = "end-pos"; private static Logger logger=Logger.getLogger(FixedWidthFlatFileFieldSpec.class); /** * Creates a new instance of FixedWidthFlatFileFieldSpec by using field name. */ protected FixedWidthFlatFileFieldSpec(String fieldName) { super(fieldName); } /** * Gets the starting position of the field. */ public int getStartPosition() { return this.startPosition; } /** * Gets the ending position of the field. */ public int getEndPosition() { return this.endPosition; } /** * Gets the field width. */ public int getFieldWidth() { return (endPosition-startPosition)+1; } /** * Factory method to create the FixedWidthFlatFileFieldSpec instance from * the given DOM Element representing the filed-spec element in record spec. * * @param fieldSpecElement DOM Element representing the field-spec element. * * @return Returns the FixedWidthFlatFileFieldSpec instance. * * @throws org.jmonks.batch.io.FileSpecException If field-spec * field-name values are not unique across record-spec. */ public static FixedWidthFlatFileFieldSpec createFixedWidthFlatFileFieldSpec(final Element fieldSpecElement) { logger.trace("Entering createFixedWidthFlatFileFieldSpec"); String fieldName=fieldSpecElement.getAttribute(FieldSpec.FIELD_NAME_ATTRIB_NAME); FixedWidthFlatFileFieldSpec fieldSpec=new FixedWidthFlatFileFieldSpec(fieldName); String startPosition=fieldSpecElement.getAttribute(FixedWidthFlatFileFieldSpec.START_POSITION_ATTRIB_NAME); if(startPosition!=null && !"".equals(startPosition.trim())) fieldSpec.startPosition=Integer.parseInt(startPosition); else throw new FileSpecException("start-pos attribute is required in fixed width flat file spec element."); String endPosition=fieldSpecElement.getAttribute(FixedWidthFlatFileFieldSpec.END_POSITION_ATTRIB_NAME); if(endPosition!=null && !"".equals(endPosition.trim())) fieldSpec.endPosition=Integer.parseInt(endPosition); else throw new FileSpecException("end-pos attribute is required in fixed width flat file spec element."); return fieldSpec; } /** * @see java.lang.Object#toString() */ public String toString() { StringBuffer stringValue=new StringBuffer("{FixedWidthFlatFileFieldSpec "); stringValue.append("[fieldName = " + super.fieldName + "]"); stringValue.append("[startPosition = " + this.startPosition + "]"); stringValue.append("[endPosition = " + this.endPosition+ "]"); stringValue.append("}"); return stringValue.toString(); } } --- NEW FILE: package.html --- <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> <html> <head> <title>Flat File Implementation Overview</title> </head> <body> Flat file implementation for the IO API. <p> Flat files are the common type of files being used by the enterprises to exchange the data. This package provides the implementation to read and write two flat file formats fixed width and delimited flat files. Implementation assumes that each line in a flat file corresponds to a record and there is a way to identify each record and each field in that record. </p> <p> <h3>Fixed Width Flat File Implementation</h3> Fixed Width flat files consists of set of different records where each record is starts with some value to identify the record and each record consists of set of of fields where each field starts at one position and ends at one position. <h5>Defining the file spec for fixed width flat files</h5> File spec which descibes the fixed width flat file expects the <code>file-type</code> attribute value should be "fixed-width-flat". It doesnt require any additional attributes along with the <code>file-type</code> attribute. <pre> <file-spec file-type="fixed-width-flat"> </file-spec> </pre> There could be multiple record specs exists in a file spec. Along with the <code>record-type</code> attribute, it requires two additional attributes <code>starts-with</code> which tells the value that the record should starts with and <code>record-length</code> which tells the lenght of the record. <pre> <file-spec file-type="fixed-width-flat"> <record-spec record-type="DETAIL" starts-with="5" record-length="42"> </record-spec> </file-spec> </pre> There could be multiple field specs exists in a single record spec. Field spec requires few additional attributes along with the field-name attribute to identify or extract the field from the record. These attributes are <code>start-pos</code> which tells the starting position of the field in the record and <code>end-pos</code> which tells the ending position of the field in the record. Starting index in a record starts with "1". <pre> <file-spec file-type="fixed-width-flat"> <record-spec record-type="DETAIL" starts-with="5" record-length="42"> <field-spec field-name="field-name1" start-pos="12" end-pos="23"/> </record-spec> </file-spec> </pre> <h5>Reading the records from fixed width flat files</h5> Reading the records from the fixed width flat files is fairly simple. <pre style="color:green"> <i> InputStream fileSpecInputStream= // Get the input stream for the XML file contains the file structure. InputStream fileInputStream= // Get the input stream of the file to be read. FileReader reader=FileReader.getFileReader(fileInputStream,fileSpecInputStream); ReaderRecord record=null; while((record=reader.getNextRecord())!=null) { if(record.getRecordType().equals(RecordType.DETAIL)) { String fieldValue1=record.readField("field-name1"); // Read the rest of the field and does the processing. } } reader.close(); </i> </pre> <h5>Writing the records into fixed width flat files</h5> Writing the records into fixed width flat files is fairly simple. <pre style="color:green"> <i> InputStream fileSpecInputStream= // Get the input stream for the XML file contains the file structure. OutputStream fileOutputStream= // Get the output stream of the file to be written/generated. FileWriter writer=FileWriter.getFileWriter(fileOutputStream,fileSpecInputStream); WriterRecord record=writer.createWriterRecord(RecordType.DETAIL); record.writeField("field-name1","field-value1"); // Write all the other field values into the writer record. writer.writeRecord(record); writer.close(); </i> </pre> </p> <p> <h3>Delimited Flat File Implementation</h3> Delimited flat files consists of set of records of same type where all the records consists of same number of fields and each field is are delimited by a special value in the record. <h5>Defining the file spec for delimited flat files</h5> File spec which descibes the delimited flat file expects the <code>file-type</code> attribute value should be "delimited-flat". It doesnt require any additional attributes along with the <code>file-type</code> attribute. <pre> <file-spec file-type="delimited-flat"> </file-spec> </pre> Since all the records in delimited flat file of the same type, it allows only one record spec should exists in the file spec. Along with the record-type attribute, it requires two additional attributes <code>delimiter</code> and <code>field-count</code>. Attributes <code>delimiter</code> tells the value that delimites fields among the record and attribute <code>field-count</code> tells the number of fields can exists in the record. <pre> <file-spec file-type="delimited-flat"> <record-spec record-type="DETAIL" delimiter="|" field-count="4"> </record-spec> </file-spec> </pre> There could be multiple field specs exists in a record spec. Field spec requires one additional attributes along with the field-name attribute to identify or extract the field from the record. This attribute is <code>index</code> which tells the position of the field in the record. Starting index in a record starts with "1". <pre> <file-spec file-type="delimited-flat"> <record-spec record-type="DETAIL" delimiter="|" field-count="4"> <field-spec field-name="field-name1" index="3"/> </record-spec> </file-spec> </pre> <h5>Reading the records from delimited flat files</h5> Reading the records from the delimited flat files is fairly simple. <pre style="color:green"> <i> InputStream fileSpecInputStream= // Get the input stream for the XML file contains the file structure. InputStream fileInputStream= // Get the input stream of the file to be read. FileReader reader=FileReader.getFileReader(fileInputStream,fileSpecInputStream); ReaderRecord record=null; while((record=reader.getNextRecord())!=null) { if(record.getRecordType().equals(RecordType.DETAIL)) { String fieldValue1=record.readField("field-name1"); // Read the rest of the field and does the processing. } } reader.close(); </i> </pre> <h5>Writing the records into delimited flat files</h5> Writing the records into delimited flat files is fairly simple. <pre style="color:green"> <i> InputStream fileSpecInputStream= // Get the input stream for the XML file contains the file structure. OutputStream fileOutputStream= // Get the output stream of the file to be written/generated. FileWriter writer=FileWriter.getFileWriter(fileOutputStream,fileSpecInputStream); WriterRecord record=writer.createWriterRecord(RecordType.DETAIL); record.writeField("field-name1","field-value1"); // Write all the other field values into the writer record. writer.writeRecord(record); writer.close(); </i> </pre> </p> </body> </html> --- NEW FILE: FlatFileWriter.java --- /* * FixedWidthFlatFileWriter.java * * Created on June 6, 2006, 8:58 AM * * To change this template, choose Tools | Options and locate the template under * the Source Creation and Management node. Right-click the template and choose * Open. You can then make changes to the template in the Source Editor. */ package org.jmonks.batch.io.flat; import java.io.BufferedWriter; import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; import java.util.HashMap; import java.util.Map; import org.apache.log4j.Logger; import org.jmonks.batch.io.FileParseException; import org.jmonks.batch.io.FileSpec; import org.jmonks.batch.io.FileWriter; import org.jmonks.batch.io.RecordType; import org.jmonks.batch.io.WriterRecord; /** * <p> * FlatFileWriter writes/generates the file according to the given file spec * with the data submitted in the form of WriterRecord's. This provides the methods * to create the required writer records and write the writer record into the file writer. * Once finished writing of all the records file writer should be closed by calling * the "close" method. * </p> * * @author Suresh Pragada * @version 1.0 * @since 1.0 */ public abstract class FlatFileWriter extends FileWriter { /** * File spec to be used in writing the file. */ protected FileSpec fileSpec=null; /** * Buffered writer to conveniently write each record as a new line into the file. */ private BufferedWriter writer=null; /** * Flag holds whethe first record has been written or not. This is * to identify whether we need to write EOL character before every record. */ private boolean writtenFirstLine=false; private static Logger logger=Logger.getLogger(FlatFileWriter.class); /** * Constructs and initializes the writer with the given values. * * @param outputStream Output stream to write the records. * @param fileSpec File spec to be used to generate the file. */ public FlatFileWriter(OutputStream outputStream,FileSpec fileSpec) { this(new OutputStreamWriter(outputStream), fileSpec); } /** * Constructs and initializes the writer with the given values. * * @param writer Writer to write the records. * @param fileSpec File spec to be used to generate the file. */ public FlatFileWriter(Writer writer,FileSpec fileSpec) { logger.trace("Entering FlatFileWriter constructor"); this.fileSpec=fileSpec; this.writer=new BufferedWriter(writer); logger.info("Done creating the flat file writer."); } /** * Creates the writer record assocites with the given record type. * IllegalArgumentException will be thrown, if there is no record * spec is found with this record type. * * @param recordType Type fo the record to be created. * * @return Returns the requested writer record. * * @throws IllegalArgumentException No record spec is found with this record type. */ public WriterRecord createWriterRecord(RecordType recordType) { if(this.fileSpec.isValidRecordType(recordType)) return new FlatFileWriterRecord(recordType); else throw new IllegalArgumentException("No record spec has been found with the record type = " + recordType); } /** * Generates the string representation of the record from given * writer record. * * @param writerRecord Record where field values can be read. * * @return Returns the string representation of the record. */ protected abstract String generateRecord(WriterRecord writerRecord) throws IOException; /** * Writes the given record into the file/writer. * * @param writerRecord Record consist of field names and values. * * @throws IllegalStateException If writer is closed and trying to write the recrod. * @throws IllegalArgumentException If writer record is null. * @throws org.jmonks.batch.io.FileParseException Problems while trying to write the record. */ public void writeRecord(WriterRecord writerRecord) { logger.trace("Entering writeRecord"); if(writerRecord!=null) { if(this.writer!=null) { try { String record=generateRecord(writerRecord); if(writtenFirstLine) this.writer.newLine(); else writtenFirstLine=true; this.writer.write(record); } catch(IOException exception) { exception.printStackTrace(); logger.fatal("IOException while writing the record into the reader. Message = " + exception.getMessage(),exception); throw new FileParseException("IOException while creating the reader. Message = " + exception.getMessage()); } } else throw new IllegalStateException("FileWriter is not available to write this record. Writer is either closed or not initialized properly."); } else throw new IllegalArgumentException("Writer record cannot be null to write record into the file."); logger.trace("Exiting writeRecord"); } /** * Closes the writer and all the resource being used by this writer. * This includes the output stream passed to construct the writer. */ public void close() { logger.trace("Entering close"); if(this.writer!=null) { try { this.writer.close(); logger.debug("Writer has been closed"); } catch(IOException exception) { logger.debug("IOException while closing the reader. Message = " + exception.getMessage(),exception); } finally { this.writer=null; } } } /** * FixedWidthFlatFileWriterRecord implements WriterRecord by maintaing the * field names and values as a map and provides the methods with proper * access privileges to read into and write from the record. * * @author Suresh Pragada */ public class FlatFileWriterRecord extends WriterRecord { /** * Map to hold the field names and values. */ private Map fieldMap=null; /** * Constructs and initializes the writer record. */ protected FlatFileWriterRecord(RecordType recordType) { super(recordType); fieldMap=new HashMap(); } /** * Writes the field data into the record. * * @param fieldName Name of the field defined in field spec. * @param fieldValue Value for the field name. */ public void writeField(String fieldName, Object fieldValue) { this.fieldMap.put(fieldName,fieldValue); } /** * Reads and returns the value associated with requested field name. * * @param fieldName Name of the field. * * @return Returns the value associated with the field name. */ protected Object readField(String fieldName) { if(this.fieldMap.containsKey(fieldName)) return this.fieldMap.get(fieldName); else return ""; } } } --- NEW FILE: FixedWidthFlatFileRecordSpec.java --- /* * FixedWidthFlatFileRecordSpec.java * * Created on May 26, 2006, 11:33 PM * * To change this template, choose Tools | Options and locate the template under * the Source Creation and Management node. Right-click the template and choose * Open. You can then make changes to the template in the Source Editor. */ package org.jmonks.batch.io.flat; import java.util.Iterator; import org.apache.log4j.Logger; import org.w3c.dom.Element; import org.w3c.dom.NodeList; import org.jmonks.batch.io.RecordType; import org.jmonks.batch.io.FieldSpec; import org.jmonks.batch.io.RecordSpec; import org.jmonks.batch.io.FileSpecException; /** * <p> * FixedWidthFlatFileRecordSpec represents record-spec element in the file spec * belongs to the fixed width flat file type. Along with the <code>record-type</code> attribute * it looks for the <code>starts-with</code> and <code>record-length</code> attributes * in the record-spec element to identify the record. <code>starts-with</code> tells * the value that the record starts with and <code>record-length</code> tells the * length of the record. Record type and starts-with attribute values * should be unique across the file-spec. Here is a sample spec snippet... * </p> * <p> * <pre> * <file-spec file-type="fixed-width-flat"> * <record-spec record-type="detail" starts-with="5" record-length="42"> * <!-- field specs will follow here --> * </record-spec> * </file-spec> * </pre> * </p> * * @author Suresh Pragada * @version 1.0 * @since 1.0 */ public class FixedWidthFlatFileRecordSpec extends RecordSpec { /** * Holds the record specs starts with attribute value. */ protected String startsWith=null; /** * Holds the length of the record. */ protected int recordLength=0; /** * Constant defines the starts with attribute name. */ public static final String STARTS_WITH_ATTRIB_NAME = "starts-with"; /** * Constant defines the record length attribute name. */ public static final String RECORD_LENGTH_ATTRIB_NAME = "record-length"; private static Logger logger=Logger.getLogger(FixedWidthFlatFileRecordSpec.class); /** * Constructs the record spec by accepting the record type. */ protected FixedWidthFlatFileRecordSpec(RecordType recordType) { super(recordType); } /** * Gets the starts-with value of this record. */ public String getStartsWith() { return this.startsWith; } /** * Gets the length of the record. This will be obtained from the record-legnth * attribute in the record-spec. This will be used while writing/generating the record using * file writer and while reading the record to identify the record type. */ public int getRecordLength() { return recordLength; } /** * Tells whether the given record matches with the record spec. It checks whether * this record string starts with the starts-with value and the length is equals * to the record length. * * @param recordString Record read from the file. * * @return Returns true if the recordString matches this recrd spec, false otherwise. */ public boolean isMatch(String recordString) { if(recordString==null) throw new IllegalArgumentException("Record string cannot be null to match record spec."); return recordString.startsWith(this.startsWith) && recordString.length()==recordLength; } /** * Factory method to create the fixed width flat record spec from the given * DOM Element representing the record-spec element in the file spec. * * @param recordSpecElement DOM Element representing the record spec. * * @return Returns the fixed width flat file record spec. * * @throws org.jmonks.batch.io.FileSpecException If record-spec * doesnt have the starts-with or record-type attribute. */ public static FixedWidthFlatFileRecordSpec createFixedWidthFlatFileRecordSpec(final Element recordSpecElement) { logger.trace("Entering createFixedWidthFlatFileRecordSpec"); FixedWidthFlatFileRecordSpec recordSpec=null; String configuredRecordType=recordSpecElement.getAttribute(RecordSpec.RECORD_TYPE_ATTRIB_NAME); logger.debug("record specs record-type value = " + configuredRecordType); if(configuredRecordType!=null && !(configuredRecordType.trim().equals(""))) { RecordType recordType=RecordType.toRecordType(configuredRecordType); recordSpec=new FixedWidthFlatFileRecordSpec(recordType); } else throw new FileSpecException("Record Spec in Fixed Width File Spec should have record-type attribute."); String startsWith=recordSpecElement.getAttribute(FixedWidthFlatFileRecordSpec.STARTS_WITH_ATTRIB_NAME); logger.debug("record specs starts-with value = " + startsWith); if(startsWith!=null) // && !"".equals(startsWith.trim())) recordSpec.startsWith=startsWith.trim(); else throw new FileSpecException("Record Spec in Fixed Width File Spec should have starts-with attribute."); String recordLength=recordSpecElement.getAttribute(FixedWidthFlatFileRecordSpec.RECORD_LENGTH_ATTRIB_NAME); logger.debug("record specs length value = " + startsWith); if(recordLength!=null && !"".equals(startsWith.trim())) recordSpec.recordLength=Integer.parseInt(recordLength); else throw new FileSpecException("Record Spec in Fixed Width File Spec should have record-length attribute."); NodeList fieldSpecNodeList=recordSpecElement.getElementsByTagName(FieldSpec.FIELD_SPEC_TAG_NAME); for(int i=0;i<fieldSpecNodeList.getLength();i++) { FixedWidthFlatFileFieldSpec fieldSpec=FixedWidthFlatFileFieldSpec.createFixedWidthFlatFileFieldSpec((Element)fieldSpecNodeList.item(i)); recordSpec.addFieldSpec(fieldSpec); } return recordSpec; } /** * @see java.lang.Object#toString() */ public String toString() { StringBuffer stringValue=new StringBuffer("{FixedWidthFlatFileRecordSpec "); stringValue.append("[recordType = " + super.recordType.toString() + "]"); stringValue.append("[startsWith = " + this.startsWith + "]"); stringValue.append("[recordLength = " + this.recordLength + "]"); stringValue.append("[fieldSpecList = "); for(Iterator iterator=fieldSpecList.iterator();iterator.hasNext();) stringValue.append(((FixedWidthFlatFileFieldSpec)iterator.next()).toString()); stringValue.append("]}"); return stringValue.toString(); } } --- NEW FILE: sample-fixed-width-file-spec.xml --- <?xml version="1.0" encoding="UTF-8"?> <file-spec file-type="fixed-width-flat"> <record-spec record-type="header" starts-with="1" record-length="9"> <field-spec field-name="timestamp" start-pos="2" end-pos="9"/> </record-spec> <record-spec record-type="detail" starts-with="5" record-length="41"> <field-spec field-name="field1" start-pos="2" end-pos="11"/> <field-spec field-name="field2" start-pos="12" end-pos="21"/> <field-spec field-name="field3" start-pos="22" end-pos="31"/> <field-spec field-name="field4" start-pos="32" end-pos="41"/> </record-spec> <record-spec record-type="trailer" starts-with="6" record-length="9"> <field-spec field-name="recordCount" start-pos="2" end-pos="9"/> </record-spec> </file-spec> --- NEW FILE: sample-fixed-width-file.dat --- 120060526 512345678910000012.0034343434347878787878 512345678920000013.0034343434347878787878 512345678930000014.0034343434347878787878 512345678940000015.0034343434347878787878 512345678950000016.0034343434347878787878 60000000.00 --- NEW FILE: sample-delimited-file.dat --- 1234567891|0000012.00|3434343434|7878787878 1234567892|0000013.00|3434343434|7878787878 1234567893|0000014.00|3434343434|7878787878 1234567894|0000015.00|3434343434|7878787878 1234567895|0000016.00|3434343434|7878787878 --- NEW FILE: DelimitedFlatFileFieldSpec.java --- /* * DelimitedFlatFileFieldSpec.java * * Created on June 10, 2006, 11:09 AM * * To change this template, choose Tools | Options and locate the template under * the Source Creation and Management node. Right-click the template and choose * Open. You can then make changes to the template in the Source Editor. */ package org.jmonks.batch.io.flat; import org.apache.log4j.Logger; import org.jmonks.batch.io.FieldSpec; import org.jmonks.batch.io.FileSpecException; import org.w3c.dom.Element; /** * <p> * DelimitedFlatFileFieldSpec represents <code>field-spec</code> element in the <code>record spec</code> * element belongs to the delimited flat file type. Along with the <code>field-name</code> attribute * it looks for the <code>index</code> attribute in the field-spec element to retrieve * the field from the record. It specifies the field position in the record. * Field index value starts with 1. * Filed name on field-spec should be unique across the record-spec. * Here is a sample spec snippet... * </p> * <p> * <pre> * <file-spec file-type="delimited-flat"> * <record-spec record-type="detail" delimiter="|"> * <field-spec field-name="consumer-id" index="1"/> * <field-spec field-name="consumer-name" index="2"/> * </record-spec> * </file-spec> * </pre> * </p> * * @author Suresh Pragada * @version 1.0 * @since 1.0 */ public class DelimitedFlatFileFieldSpec extends FieldSpec { /** * Represents the field position in the record. */ private int index; /** * Constant defines the index attribute name which is "index" */ public static final String INDEX_ATTRIB_NAME = "index"; private static Logger logger=Logger.getLogger(DelimitedFlatFileFieldSpec.class); /** * Creates a new instance of DelimitedFlatFileFieldSpec by using field name. */ protected DelimitedFlatFileFieldSpec(String fieldName) { super(fieldName); } /** * Returns the index of field in the record. Index value starts with 1. */ public int getIndex() { return this.index; } /** * Factory method to create the DelimitedFlatFileFieldSpec instance from * the given DOM Element representing the filed-spec element in record spec. * * @param fieldSpecElement DOM Element representing the field-spec element. * * @return Returns the DelimitedFlatFileFieldSpec instance. * * @throws org.jmonks.batch.io.FileSpecException If field-spec * field-name values are not unique across record-spec and it is missing index attribute. */ public static DelimitedFlatFileFieldSpec createDelimitedFlatFileFieldSpec(final Element fieldSpecElement) { logger.trace("Entering createDelimitedFlatFileFieldSpec"); String fieldName=fieldSpecElement.getAttribute(FieldSpec.FIELD_NAME_ATTRIB_NAME); DelimitedFlatFileFieldSpec fieldSpec=new DelimitedFlatFileFieldSpec(fieldName); String index=fieldSpecElement.getAttribute(DelimitedFlatFileFieldSpec.INDEX_ATTRIB_NAME); if(index!=null && !"".equals(index.trim())) fieldSpec.index=Integer.parseI... [truncated message content] |